From ea3131b97f88b145cf1645706639e0f545323ac1 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 26 Apr 2023 20:39:16 +0200 Subject: [PATCH 01/37] feat(inv): introduce OmegaConf as optional inventory backend --- kapitan/cli.py | 25 +++++++++ kapitan/refs/cmd_parser.py | 24 ++++---- kapitan/resources.py | 109 +++++++++++++++++++++++++++++++++---- kapitan/targets.py | 11 ++-- poetry.lock | 29 +++++++++- pyproject.toml | 1 + tests/test_compile.py | 4 +- 7 files changed, 173 insertions(+), 30 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index a4fca3225..5a4beb945 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -306,6 +306,13 @@ def build_parser(): help="dumps all none-type entries as empty, default is dumping as 'null'", ) + compile_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) + compile_selector_parser = compile_parser.add_mutually_exclusive_group() compile_selector_parser.add_argument( "--targets", @@ -379,6 +386,12 @@ def build_parser(): default=from_dot_kapitan("inventory", "multiline-string-style", "double-quotes"), help="set multiline string style to STYLE, default is 'double-quotes'", ) + inventory_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) searchvar_parser = subparser.add_parser( "searchvar", aliases=["sv"], help="show all inventory files where var is declared" @@ -505,6 +518,12 @@ def build_parser(): action="store_true", default=from_dot_kapitan("refs", "verbose", False), ) + refs_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) lint_parser = subparser.add_parser("lint", aliases=["l"], help="linter for inventory and refs") lint_parser.set_defaults(func=start_lint, name="lint") @@ -548,6 +567,12 @@ def build_parser(): default=from_dot_kapitan("lint", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) + lint_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) init_parser = subparser.add_parser( "init", help="initialize a directory with the recommended kapitan project skeleton." diff --git a/kapitan/refs/cmd_parser.py b/kapitan/refs/cmd_parser.py index f94496376..5cedeb751 100644 --- a/kapitan/refs/cmd_parser.py +++ b/kapitan/refs/cmd_parser.py @@ -16,7 +16,7 @@ from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints from kapitan.refs.secrets.vaultkv import VaultSecret from kapitan.refs.secrets.vaulttransit import VaultTransit -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.utils import fatal_error, search_target_token_paths logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -95,7 +95,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -123,7 +123,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -152,7 +152,7 @@ def ref_write(args, ref_controller): type_name, token_path = token_name.split(":") key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -196,7 +196,7 @@ def ref_write(args, ref_controller): vault_params = {} encoding = "original" if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError( @@ -230,7 +230,7 @@ def ref_write(args, ref_controller): _data = data.encode() vault_params = {} if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -302,7 +302,7 @@ def secret_update(args, ref_controller): for name in args.recipients ] if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -330,7 +330,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("gkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -356,7 +356,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("azkms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -382,7 +382,7 @@ def secret_update(args, ref_controller): elif token_name.startswith("awskms:"): key = args.key if args.target_name: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"] if "secrets" not in kap_inv_params: raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name)) @@ -439,7 +439,7 @@ def secret_update_validate(args, ref_controller): "Validate and/or update target secrets" # update gpg recipients/gkms/awskms key for all secrets in secrets_path # use --refs-path to set scanning path - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) targets = set(inv["nodes"].keys()) secrets_path = os.path.abspath(args.refs_path) target_token_paths = search_target_token_paths(secrets_path, targets) diff --git a/kapitan/resources.py b/kapitan/resources.py index e77f4c885..99fe7af42 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -26,6 +26,7 @@ import reclass import reclass.core from reclass.errors import NotFoundError, ReclassException +from omegaconf import OmegaConf, errors logger = logging.getLogger(__name__) @@ -280,14 +281,14 @@ def inventory(search_paths, target, inventory_path=None): raise InventoryError(f"Inventory not found in search paths: {search_paths}") if target is None: - return inventory_reclass(full_inv_path)["nodes"] + return get_inventory(full_inv_path)["nodes"] - return inventory_reclass(full_inv_path)["nodes"][target] + return get_inventory(full_inv_path)["nodes"][target] def generate_inventory(args): try: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) if args.target_name != "": inv = inv["nodes"][args.target_name] if args.pattern != "": @@ -304,6 +305,31 @@ def generate_inventory(args): sys.exit(1) +def get_inventory(inventory_path, ignore_class_notfound=False): + """ + generic inventory function that makes inventory backend pluggable + default backend is reclass + """ + + # if inventory is already cached theres nothing to do + if cached.inv: + return cached.inv + + # get parsed args from cached.py + args = list(cached.args.values())[0] + use_omegaconf = args.omegaconf + + if use_omegaconf: + logger.debug("Using omegaconf as inventory backend") + inv = inventory_omegaconf(inventory_path) + else: + logger.debug("Using reclass as inventory backend") + inv = inventory_reclass(inventory_path, ignore_class_notfound) + + cached.inv = inv + return inv + + def inventory_reclass(inventory_path, ignore_class_notfound=False): """ Runs a reclass inventory in inventory_path @@ -314,12 +340,8 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): Does not throw errors if a class is not found while --fetch flag is enabled """ - # if inventory is already cached theres nothing to do - if cached.inv: - return cached.inv - # set default values initially - reclass_config = reclass_config_defaults = { + reclass_config = { "storage_type": "yaml_fs", "inventory_base_uri": inventory_path, "nodes_uri": "targets", @@ -358,7 +380,7 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - cached.inv = _reclass.inventory() + return _reclass.inventory() except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") @@ -366,4 +388,71 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): logger.error("Inventory reclass error: %s", e.message) raise InventoryError(e.message) - return cached.inv + +def inventory_omegaconf(inventory_path, ignore_class_notfound=False): + """ + generates inventory from yaml files using OmegaConf + """ + + # TBD: make log output pretty + logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") + + # TBD: add config to specify paths + targets_path = os.path.join(inventory_path, "targets") + classes_path = os.path.join(inventory_path, "classes") + + target_configs = {} + # load targets + # TBD: big opportunity to boost the performance with flag '-t' because we're able to skip targets directly + for root, dirs, files in os.walk(targets_path): + for target_name in files: + target_path = os.path.join(root, target_name) + target_config = OmegaConf.load(target_path) + + target_config_classes = target_config.pop("classes", []) + + # load classes for targets + for class_name in target_config_classes: + + # resolve class name (relative paths TBD) + class_path = os.path.join(classes_path, *class_name.split(".")) + ".yml" + if os.path.isfile(class_path): + # load classes recursively + class_config = OmegaConf.load(class_path) + + target_config_classes.extend(class_config.pop("classes", [])) + + elif not ignore_class_notfound: + raise InventoryError(f"Target: {target_name}: Class {class_name} not found.") + else: + continue + + # merge target with loaded classes + target_config = OmegaConf.merge(target_config, class_config) + + # resolve references / interpolate values + try: + target_config = OmegaConf.to_container(target_config, resolve=True) + except errors.InterpolationKeyError as e: + raise InventoryError(f"{target_name}: {e.__context__}") + + # obtain target name to insert in inv dict (legacy) (refactoring TBD) + try: + target_name = target_config["parameters"]["kapitan"]["vars"]["target"] + except KeyError: + logger.warning(f"Could not resolve target name on target {target_name}") + target_name = os.path.splitext(target_name) + + # append meta data _reclass_ (legacy) (refactoring TBD) + # TBD: compose_node_name integration / behavior + target_config["parameters"]["_reclass_"] = { + "name": {"full": target_name, "path": target_name, "short": target_name} + } + + # print(OmegaConf.to_yaml(target_config)) + target_configs[target_name] = target_config + + # TBD: refactor inventory accessing (targets.py, cmd_parser.py) + inv = {} + inv["nodes"] = target_configs + return inv diff --git a/kapitan/targets.py b/kapitan/targets.py index 7327fb7d8..b762c76c0 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -32,7 +32,7 @@ from kapitan.inputs.kadet import Kadet from kapitan.inputs.remove import Remove from kapitan.remoteinventory.fetch import fetch_inventories, list_sources -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache from kapitan.validator.kubernetes_validator import KubernetesManifestValidator @@ -241,7 +241,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): ... } """ - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) cached.inv_cache = {} cached.inv_cache["inventory"] = {} cached.inv_cache["folder"] = {} @@ -295,7 +295,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): def changed_targets(inventory_path, output_path): """returns a list of targets that have changed since last compilation""" targets = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) saved_inv_cache = None saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache") @@ -388,7 +388,7 @@ def save_inv_cache(compile_path, targets): def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): """returns a list of target objects from the inventory""" target_objs = [] - inv = inventory_reclass(inventory_path, ignore_class_notfound) + inv = get_inventory(inventory_path, ignore_class_notfound) # if '-t' is set on compile, only loop through selected targets if targets: @@ -399,6 +399,7 @@ def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): for target_name in targets_list: try: inv_target = inv["nodes"][target_name] + print(yaml.dump(inv_target)) target_obj = inv_target["parameters"]["kapitan"] # check if parameters.kapitan is empty if not target_obj: @@ -431,7 +432,7 @@ def search_targets(inventory_path, targets, labels): ) targets_found = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) for target_name in inv["nodes"]: matched_all_labels = False diff --git a/poetry.lock b/poetry.lock index 5a8e6a21c..f7473ee3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,6 +12,17 @@ files = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] + [[package]] name = "attrs" version = "22.2.0" @@ -817,6 +828,22 @@ portalocker = [ {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, ] +[[package]] +name = "omegaconf" +version = "2.3.0" +description = "A flexible configuration library" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, + {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, +] + +[package.dependencies] +antlr4-python3-runtime = ">=4.9.0,<4.10.0" +PyYAML = ">=5.1.0" + [[package]] name = "packaging" version = "23.0" @@ -1430,4 +1457,4 @@ test = ["docker"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "290a88e4534ce117f650cde4054b67ea83ab345e8256c7dcd40ee3dc9b88781a" +content-hash = "c1e54c3c6cb146197e2bb37013bf758b9dd25bcb3c8dfc48c553823478264101" diff --git a/pyproject.toml b/pyproject.toml index 8329fbffb..14df9c424 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ packaging = "^23.0" typing-extensions = "^4.0.0" gojsonnet = { version = "^0.17.0", optional = true } docker = { version = "^5.0.0", optional = true } +omegaconf = "^2.3.0" [tool.poetry.extras] gojsonnet = ["gojsonnet"] diff --git a/tests/test_compile.py b/tests/test_compile.py index 5196ae99b..f65e209dd 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -20,7 +20,7 @@ from kapitan.utils import directory_hash from kapitan.cached import reset_cache from kapitan.targets import validate_matching_target_name -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.errors import InventoryError @@ -175,7 +175,7 @@ def test_compile_not_matching_targets(self): def test_compile_vars_target_missing(self): inventory_path = "inventory" target_filename = "minikube-es" - target_obj = inventory_reclass(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"] + target_obj = get_inventory(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"] # delete vars.target del target_obj["vars"]["target"] From 9c57cae531984e30b7d930caf0c80b6e44a39cfc Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:25:07 +0200 Subject: [PATCH 02/37] remove print statement --- kapitan/targets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kapitan/targets.py b/kapitan/targets.py index b762c76c0..473fd965c 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -399,7 +399,6 @@ def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): for target_name in targets_list: try: inv_target = inv["nodes"][target_name] - print(yaml.dump(inv_target)) target_obj = inv_target["parameters"]["kapitan"] # check if parameters.kapitan is empty if not target_obj: From 770404c3c734a27cb534c560ea44e861946f0873 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:48:24 +0200 Subject: [PATCH 03/37] fix(lint): fix linting --- kapitan/resources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kapitan/resources.py b/kapitan/resources.py index 99fe7af42..2e51e411d 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -413,7 +413,6 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False): # load classes for targets for class_name in target_config_classes: - # resolve class name (relative paths TBD) class_path = os.path.join(classes_path, *class_name.split(".")) + ".yml" if os.path.isfile(class_path): From 083160f27e3e4ad820d6d4b6865baaf5cbc55827 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 10 May 2023 17:51:59 +0200 Subject: [PATCH 04/37] add first version of migration script --- kapitan/migrate_omegaconf.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 kapitan/migrate_omegaconf.py diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py new file mode 100644 index 000000000..d415c39ba --- /dev/null +++ b/kapitan/migrate_omegaconf.py @@ -0,0 +1,56 @@ +import sys +import os +from ruamel.yaml import YAML +from pathlib import Path +import regex + +REF_TOKEN = r".*\$\{.*\}.*" + + +def replace_token(token: str) -> str: + token = token.replace(":", ".") + + return token.replace + + +def replace_str(input: str) -> str: + if regex.match(REF_TOKEN, input): + return replace_token(input) + + return input + + +def migrate_file(input_file: str) -> None: + # load the file + yaml = YAML() + file_path = Path(input_file) + yaml.load(file_path) + + # replace all references with modern + + +def migrate(inv_path: str = "") -> None: + if len(sys.argv) != 2: + print("Usage: migrate_omegaconf.py INV_PATH") + return 0 + + inv_path = sys.argv[1] + + if not os.path.exists(inv_path): + print("Path does not exist") + + targets_path = os.path.join(inv_path, "targets") + classes_path = os.path.join(inv_path, "classes") + + for root, subdirs, files in os.walk(targets_path): + for target_file in files: + migrate_file(target_file) + + for root, subdirs, files in os.walk(classes_path): + for class_file in files: + migrate_file(class_file) + + +# support running the file without kapitan +if __name__ == "__main__": + migrate() From d9a1d23366858981db007d7ed1124f12334978ff Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 10 May 2023 17:52:45 +0200 Subject: [PATCH 05/37] add flag behavior for `--compose-node-name` and `-t` --- kapitan/resources.py | 50 ++++++++++++++++++++++++++++++-------------- kapitan/targets.py | 2 +- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/kapitan/resources.py b/kapitan/resources.py index 2e51e411d..4f2d0a731 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -305,7 +305,7 @@ def generate_inventory(args): sys.exit(1) -def get_inventory(inventory_path, ignore_class_notfound=False): +def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): """ generic inventory function that makes inventory backend pluggable default backend is reclass @@ -321,7 +321,7 @@ def get_inventory(inventory_path, ignore_class_notfound=False): if use_omegaconf: logger.debug("Using omegaconf as inventory backend") - inv = inventory_omegaconf(inventory_path) + inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) else: logger.debug("Using reclass as inventory backend") inv = inventory_reclass(inventory_path, ignore_class_notfound) @@ -389,7 +389,7 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): raise InventoryError(e.message) -def inventory_omegaconf(inventory_path, ignore_class_notfound=False): +def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], compose_node_name=False): """ generates inventory from yaml files using OmegaConf """ @@ -397,24 +397,45 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False): # TBD: make log output pretty logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") - # TBD: add config to specify paths - targets_path = os.path.join(inventory_path, "targets") - classes_path = os.path.join(inventory_path, "classes") + # TBD: add config to specify paths (do we need that?) + targets_searchpath = os.path.join(inventory_path, "targets") + classes_searchpath = os.path.join(inventory_path, "classes") - target_configs = {} - # load targets - # TBD: big opportunity to boost the performance with flag '-t' because we're able to skip targets directly - for root, dirs, files in os.walk(targets_path): + inv = {"nodes": {}} + + # loop through targets searchpath and load all targets + for root, dirs, files in os.walk(targets_searchpath): for target_name in files: target_path = os.path.join(root, target_name) - target_config = OmegaConf.load(target_path) + + # split file extension and check if yml/yaml + target_name, ext = os.path.splitext(target_name) + if ext not in (".yml", ".yaml"): + logger.debug(f"{target_name}: targets have to be .yml or .yaml files.") + continue + + # skip targets if they are not specified with -t flag + if targets and target_name not in targets: + continue + + # compose node name + if compose_node_name: + target_name = str(os.path.splitext(target_path)[0]).replace(targets_searchpath + os.sep, "") + target_name = target_name.replace("/", ".") + + # look for duplicate targets + if target_name in inv["nodes"]: + raise InventoryError(f"{target_name}: duplicate target are not allowed") + + # load the inventory + target_config = OmegaConf.create({"parameters": {}}) # OmegaConf.load(target_path) target_config_classes = target_config.pop("classes", []) # load classes for targets for class_name in target_config_classes: # resolve class name (relative paths TBD) - class_path = os.path.join(classes_path, *class_name.split(".")) + ".yml" + class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" if os.path.isfile(class_path): # load classes recursively class_config = OmegaConf.load(class_path) @@ -440,7 +461,6 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False): target_name = target_config["parameters"]["kapitan"]["vars"]["target"] except KeyError: logger.warning(f"Could not resolve target name on target {target_name}") - target_name = os.path.splitext(target_name) # append meta data _reclass_ (legacy) (refactoring TBD) # TBD: compose_node_name integration / behavior @@ -449,9 +469,7 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False): } # print(OmegaConf.to_yaml(target_config)) - target_configs[target_name] = target_config + inv["nodes"][target_name] = target_config # TBD: refactor inventory accessing (targets.py, cmd_parser.py) - inv = {} - inv["nodes"] = target_configs return inv diff --git a/kapitan/targets.py b/kapitan/targets.py index 473fd965c..0c7ab7fd4 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -388,7 +388,7 @@ def save_inv_cache(compile_path, targets): def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): """returns a list of target objects from the inventory""" target_objs = [] - inv = get_inventory(inventory_path, ignore_class_notfound) + inv = get_inventory(inventory_path, ignore_class_notfound, targets) # if '-t' is set on compile, only loop through selected targets if targets: From fc314f76b09921ddf636efa218db8cb4a96fabc6 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 11 May 2023 13:44:22 +0200 Subject: [PATCH 06/37] add resolving step for references in migration script --- kapitan/migrate_omegaconf.py | 59 ++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py index d415c39ba..93a9a6ad8 100644 --- a/kapitan/migrate_omegaconf.py +++ b/kapitan/migrate_omegaconf.py @@ -2,31 +2,68 @@ import os from ruamel.yaml import YAML from pathlib import Path -import regex +import regex as re -REF_TOKEN = r".*\$\{.*\}.*" +REF_TOKEN = r"\${([^\${}]*+(?:(?R)[^\${}]*)*+)}" def replace_token(token: str) -> str: - token = token.replace(":", ".") - return token.replace + inner_token = token[2:-1] + + offset = 0 + matches = re.finditer(REF_TOKEN, inner_token) + for match in matches: + replaced = replace_token(match.group()) + inner_token = inner_token[: match.start() - offset] + replaced + inner_token[match.end() - offset :] + offset += len(match.group()) - len(replaced) + + inner_token = inner_token.replace(":", ".") + inner_token = "parameters." + inner_token + + token = "${" + inner_token + "}" + + return token def replace_str(input: str) -> str: - if regex.match(REF_TOKEN, input): - return replace_token(input) + + offset = 0 + matches = re.finditer(REF_TOKEN, input) + for match in matches: + replaced = replace_token(match.group()) + input = input[: match.start() - offset] + replaced + input[match.end() - offset :] + offset += len(match.group()) - len(replaced) return input def migrate_file(input_file: str) -> None: # load the file - yaml = YAML() - file_path = Path(input_file) - yaml.load(file_path) + yaml = YAML(typ="rt") + file_path = Path(input_file).resolve() + yaml_obj = yaml.load(file_path) + yaml.dump(yaml_obj, file_path) + + print(yaml_obj) + return + + # replace all references with OmegaConf syntax + def migrate_yaml_obj(yaml_obj: dict | list | str) -> None: + # dictionary + if isinstance(yaml_obj, dict): + for k, v in yaml_obj.items(): + yaml_obj[k] = migrate_yaml_obj(v) + # list + elif isinstance(yaml_obj, list): + yaml_obj = [migrate_yaml_obj(item) for item in yaml_obj] + # string --> replace the references + elif isinstance(yaml_obj, str): + yaml_obj = replace_str(yaml_obj) + + yaml_obj = migrate_yaml_obj(yaml_obj) - # replace all references with modern + yaml.dump(yaml_obj, file_path) def migrate(inv_path: str = "") -> None: @@ -44,7 +81,9 @@ def migrate(inv_path: str = "") -> None: for root, subdirs, files in os.walk(targets_path): for target_file in files: + target_file = os.path.join(root, target_file) migrate_file(target_file) + return for root, subdirs, files in os.walk(classes_path): for class_file in files: From 0e76778ec1e653a457a7c4abfe650a8bec772447 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 17 May 2023 14:27:44 +0200 Subject: [PATCH 07/37] feat: finish migration script --- kapitan/migrate_omegaconf.py | 67 ++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py index 93a9a6ad8..a7fa3f1e8 100644 --- a/kapitan/migrate_omegaconf.py +++ b/kapitan/migrate_omegaconf.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import sys import os from ruamel.yaml import YAML @@ -8,9 +10,11 @@ def replace_token(token: str) -> str: - inner_token = token[2:-1] + if "parameters." in inner_token: + return token + offset = 0 matches = re.finditer(REF_TOKEN, inner_token) for match in matches: @@ -27,7 +31,6 @@ def replace_token(token: str) -> str: def replace_str(input: str) -> str: - offset = 0 matches = re.finditer(REF_TOKEN, input) for match in matches: @@ -38,43 +41,39 @@ def replace_str(input: str) -> str: return input +# replace all references with OmegaConf syntax +def migrate_yaml_obj(yaml_obj: dict | list | str) -> None: + # dictionary + if isinstance(yaml_obj, dict): + for k, v in yaml_obj.items(): + yaml_obj[k] = migrate_yaml_obj(v) + # list + elif isinstance(yaml_obj, list): + yaml_obj = [migrate_yaml_obj(item) for item in yaml_obj] + # string --> replace the references + elif isinstance(yaml_obj, str): + yaml_obj = replace_str(yaml_obj) + + return yaml_obj + + def migrate_file(input_file: str) -> None: # load the file yaml = YAML(typ="rt") + yaml.preserve_quotes = True + file_path = Path(input_file).resolve() + yaml_obj = yaml.load(file_path) yaml.dump(yaml_obj, file_path) - print(yaml_obj) - return - - # replace all references with OmegaConf syntax - def migrate_yaml_obj(yaml_obj: dict | list | str) -> None: - # dictionary - if isinstance(yaml_obj, dict): - for k, v in yaml_obj.items(): - yaml_obj[k] = migrate_yaml_obj(v) - # list - elif isinstance(yaml_obj, list): - yaml_obj = [migrate_yaml_obj(item) for item in yaml_obj] - # string --> replace the references - elif isinstance(yaml_obj, str): - yaml_obj = replace_str(yaml_obj) - yaml_obj = migrate_yaml_obj(yaml_obj) + yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(yaml_obj, file_path) -def migrate(inv_path: str = "") -> None: - if len(sys.argv) != 2: - print("Usage: migrate_omegaconf.py INV_PATH") - return 0 - - inv_path = sys.argv[1] - - if not os.path.exists(inv_path): - print("Path does not exist") +def migrate(inv_path: str = "", output_path: str = "") -> None: targets_path = os.path.join(inv_path, "targets") classes_path = os.path.join(inv_path, "classes") @@ -83,13 +82,23 @@ def migrate(inv_path: str = "") -> None: for target_file in files: target_file = os.path.join(root, target_file) migrate_file(target_file) - return for root, subdirs, files in os.walk(classes_path): for class_file in files: + class_file = os.path.join(root, class_file) migrate_file(class_file) # support running the file without kapitan if __name__ == "__main__": - migrate() + + if len(sys.argv) != 2: + print("Usage: migrate_omegaconf.py INV_PATH") + sys.exit(1) + + inv_path = sys.argv[1] + + if not os.path.exists(inv_path): + print("Path does not exist") + + migrate(inv_path) From 742b2bbdd5e4bed02878b2ac54c0f20ff5acad51 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 17 May 2023 14:28:15 +0200 Subject: [PATCH 08/37] feat: enable migration support as query --- kapitan/resources.py | 46 ++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/kapitan/resources.py b/kapitan/resources.py index 4f2d0a731..7c654e6ec 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -22,6 +22,7 @@ from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string +from kapitan.migrate_omegaconf import migrate import reclass import reclass.core @@ -321,7 +322,26 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): if use_omegaconf: logger.debug("Using omegaconf as inventory backend") - inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) + logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") + + # migrate a reclass inventory to omegaConf + while True: + query_result = input(f"Do you want to migrate your inventory to OmegaConf? (Y/n):").lower() + if not query_result or query_result[0] == "y": + output_path = inventory_path # + "_omegaconf" + if not os.path.exists(output_path): + os.mkdir(output_path) + migrate(inventory_path, output_path) + logger.info(f"Migrated inventory to OmegaConf in {output_path}") + break + elif query_result[0] == "n": + break + else: + logger.warning("Please answer with 'yes'(y) or 'no'(n)!") + try: + inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) + except errors.OmegaConfBaseException: + raise InventoryError("") else: logger.debug("Using reclass as inventory backend") inv = inventory_reclass(inventory_path, ignore_class_notfound) @@ -393,10 +413,6 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], """ generates inventory from yaml files using OmegaConf """ - - # TBD: make log output pretty - logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") - # TBD: add config to specify paths (do we need that?) targets_searchpath = os.path.join(inventory_path, "targets") classes_searchpath = os.path.join(inventory_path, "classes") @@ -428,7 +444,7 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], raise InventoryError(f"{target_name}: duplicate target are not allowed") # load the inventory - target_config = OmegaConf.create({"parameters": {}}) # OmegaConf.load(target_path) + target_config = OmegaConf.load(target_path) target_config_classes = target_config.pop("classes", []) @@ -450,10 +466,18 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], # merge target with loaded classes target_config = OmegaConf.merge(target_config, class_config) + if not target_config: + logger.warning(f"{target_name}: empty target") + + # append meta data _reclass_ (legacy) (refactoring TBD) + target_config["parameters"]["_reclass_"] = { + "name": {"full": target_name, "path": target_name, "short": target_name} + } + # resolve references / interpolate values try: target_config = OmegaConf.to_container(target_config, resolve=True) - except errors.InterpolationKeyError as e: + except errors.OmegaConfBaseException as e: raise InventoryError(f"{target_name}: {e.__context__}") # obtain target name to insert in inv dict (legacy) (refactoring TBD) @@ -462,14 +486,8 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], except KeyError: logger.warning(f"Could not resolve target name on target {target_name}") - # append meta data _reclass_ (legacy) (refactoring TBD) - # TBD: compose_node_name integration / behavior - target_config["parameters"]["_reclass_"] = { - "name": {"full": target_name, "path": target_name, "short": target_name} - } - - # print(OmegaConf.to_yaml(target_config)) inv["nodes"][target_name] = target_config # TBD: refactor inventory accessing (targets.py, cmd_parser.py) + # that it only receives the targets and not everything return inv From 79bf1e94d3f34ac46a4e65fa3b766f062d7eb8ce Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:44:39 +0200 Subject: [PATCH 09/37] add local module omegaconf to have new features --- omegaconf/__init__.py | 65 ++ omegaconf/_impl.py | 101 ++ omegaconf/_utils.py | 1011 ++++++++++++++++ omegaconf/base.py | 947 +++++++++++++++ omegaconf/basecontainer.py | 916 +++++++++++++++ omegaconf/dictconfig.py | 727 ++++++++++++ omegaconf/errors.py | 141 +++ omegaconf/grammar/OmegaConfGrammarLexer.g4 | 137 +++ omegaconf/grammar/OmegaConfGrammarParser.g4 | 91 ++ omegaconf/grammar/__init__.py | 0 omegaconf/grammar/gen/__init__.py | 0 omegaconf/grammar_parser.py | 140 +++ omegaconf/grammar_visitor.py | 362 ++++++ omegaconf/listconfig.py | 643 +++++++++++ omegaconf/nodes.py | 513 +++++++++ omegaconf/omegaconf.py | 1144 +++++++++++++++++++ omegaconf/py.typed | 0 omegaconf/resolvers/__init__.py | 5 + omegaconf/resolvers/oc/__init__.py | 109 ++ omegaconf/resolvers/oc/dict.py | 78 ++ omegaconf/version.py | 13 + 21 files changed, 7143 insertions(+) create mode 100644 omegaconf/__init__.py create mode 100644 omegaconf/_impl.py create mode 100644 omegaconf/_utils.py create mode 100644 omegaconf/base.py create mode 100644 omegaconf/basecontainer.py create mode 100644 omegaconf/dictconfig.py create mode 100644 omegaconf/errors.py create mode 100644 omegaconf/grammar/OmegaConfGrammarLexer.g4 create mode 100644 omegaconf/grammar/OmegaConfGrammarParser.g4 create mode 100644 omegaconf/grammar/__init__.py create mode 100644 omegaconf/grammar/gen/__init__.py create mode 100644 omegaconf/grammar_parser.py create mode 100644 omegaconf/grammar_visitor.py create mode 100644 omegaconf/listconfig.py create mode 100644 omegaconf/nodes.py create mode 100644 omegaconf/omegaconf.py create mode 100644 omegaconf/py.typed create mode 100644 omegaconf/resolvers/__init__.py create mode 100644 omegaconf/resolvers/oc/__init__.py create mode 100644 omegaconf/resolvers/oc/dict.py create mode 100644 omegaconf/version.py diff --git a/omegaconf/__init__.py b/omegaconf/__init__.py new file mode 100644 index 000000000..1670cf08f --- /dev/null +++ b/omegaconf/__init__.py @@ -0,0 +1,65 @@ +from .base import Container, DictKeyType, Node, SCMode, UnionNode +from .dictconfig import DictConfig +from .errors import ( + KeyValidationError, + MissingMandatoryValue, + ReadonlyConfigError, + UnsupportedValueType, + ValidationError, +) +from .listconfig import ListConfig +from .nodes import ( + AnyNode, + BooleanNode, + BytesNode, + EnumNode, + FloatNode, + IntegerNode, + PathNode, + StringNode, + ValueNode, +) +from .omegaconf import ( + II, + MISSING, + SI, + OmegaConf, + Resolver, + flag_override, + open_dict, + read_write, +) +from .version import __version__ + +__all__ = [ + "__version__", + "MissingMandatoryValue", + "ValidationError", + "ReadonlyConfigError", + "UnsupportedValueType", + "KeyValidationError", + "Container", + "UnionNode", + "ListConfig", + "DictConfig", + "DictKeyType", + "OmegaConf", + "Resolver", + "SCMode", + "flag_override", + "read_write", + "open_dict", + "Node", + "ValueNode", + "AnyNode", + "IntegerNode", + "StringNode", + "BytesNode", + "PathNode", + "BooleanNode", + "EnumNode", + "FloatNode", + "MISSING", + "SI", + "II", +] diff --git a/omegaconf/_impl.py b/omegaconf/_impl.py new file mode 100644 index 000000000..0c30ef6a2 --- /dev/null +++ b/omegaconf/_impl.py @@ -0,0 +1,101 @@ +from typing import Any + +from omegaconf import MISSING, Container, DictConfig, ListConfig, Node, ValueNode +from omegaconf.errors import ConfigTypeError, InterpolationToMissingValueError + +from ._utils import _DEFAULT_MARKER_, _get_value + + +def _resolve_container_value(cfg: Container, key: Any) -> None: + node = cfg._get_child(key) + assert isinstance(node, Node) + if node._is_interpolation(): + try: + resolved = node._dereference_node() + except InterpolationToMissingValueError: + node._set_value(MISSING) + else: + if isinstance(resolved, Container): + _resolve(resolved) + if isinstance(resolved, Container) and isinstance(node, ValueNode): + cfg[key] = resolved + else: + node._set_value(_get_value(resolved)) + else: + _resolve(node) + + +def _resolve(cfg: Node) -> Node: + assert isinstance(cfg, Node) + if cfg._is_interpolation(): + try: + resolved = cfg._dereference_node() + except InterpolationToMissingValueError: + cfg._set_value(MISSING) + else: + cfg._set_value(resolved._value()) + + if isinstance(cfg, DictConfig): + for k in cfg.keys(): + _resolve_container_value(cfg, k) + + elif isinstance(cfg, ListConfig): + for i in range(len(cfg)): + _resolve_container_value(cfg, i) + + return cfg + + +def select_value( + cfg: Container, + key: str, + *, + default: Any = _DEFAULT_MARKER_, + throw_on_resolution_failure: bool = True, + throw_on_missing: bool = False, + absolute_key: bool = False, +) -> Any: + node = select_node( + cfg=cfg, + key=key, + throw_on_resolution_failure=throw_on_resolution_failure, + throw_on_missing=throw_on_missing, + absolute_key=absolute_key, + ) + + node_not_found = node is None + if node_not_found or node._is_missing(): + if default is not _DEFAULT_MARKER_: + return default + else: + return None + + return _get_value(node) + + +def select_node( + cfg: Container, + key: str, + *, + throw_on_resolution_failure: bool = True, + throw_on_missing: bool = False, + absolute_key: bool = False, +) -> Any: + try: + # for non relative keys, the interpretation can be: + # 1. relative to cfg + # 2. relative to the config root + # This is controlled by the absolute_key flag. By default, such keys are relative to cfg. + if not absolute_key and not key.startswith("."): + key = f".{key}" + + cfg, key = cfg._resolve_key_and_root(key) + _root, _last_key, node = cfg._select_impl( + key, + throw_on_missing=throw_on_missing, + throw_on_resolution_failure=throw_on_resolution_failure, + ) + except ConfigTypeError: + return None + + return node diff --git a/omegaconf/_utils.py b/omegaconf/_utils.py new file mode 100644 index 000000000..3339c1a21 --- /dev/null +++ b/omegaconf/_utils.py @@ -0,0 +1,1011 @@ +import copy +import os +import pathlib +import re +import string +import sys +import types +import warnings +from contextlib import contextmanager +from enum import Enum +from textwrap import dedent +from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Tuple, + Type, + Union, + get_type_hints, +) + +import yaml + +from .errors import ( + ConfigIndexError, + ConfigTypeError, + ConfigValueError, + GrammarParseError, + OmegaConfBaseException, + ValidationError, +) +from .grammar_parser import SIMPLE_INTERPOLATION_PATTERN, parse + +try: + import dataclasses + +except ImportError: # pragma: no cover + dataclasses = None # type: ignore # pragma: no cover + +try: + import attr + +except ImportError: # pragma: no cover + attr = None # type: ignore # pragma: no cover + +NoneType: Type[None] = type(None) + +BUILTIN_VALUE_TYPES: Tuple[Type[Any], ...] = ( + int, + float, + bool, + str, + bytes, + NoneType, +) + +# Regexprs to match key paths like: a.b, a[b], ..a[c].d, etc. +# We begin by matching the head (in these examples: a, a, ..a). +# This can be read as "dots followed by any character but `.` or `[`" +# Note that a key starting with brackets, like [a], is purposedly *not* +# matched here and will instead be handled in the next regex below (this +# is to keep this regex simple). +KEY_PATH_HEAD = re.compile(r"(\.)*[^.[]*") +# Then we match other keys. The following expression matches one key and can +# be read as a choice between two syntaxes: +# - `.` followed by anything except `.` or `[` (ex: .b, .d) +# - `[` followed by anything then `]` (ex: [b], [c]) +KEY_PATH_OTHER = re.compile(r"\.([^.[]*)|\[(.*?)\]") + + +# source: https://yaml.org/type/bool.html +YAML_BOOL_TYPES = [ + "y", + "Y", + "yes", + "Yes", + "YES", + "n", + "N", + "no", + "No", + "NO", + "true", + "True", + "TRUE", + "false", + "False", + "FALSE", + "on", + "On", + "ON", + "off", + "Off", + "OFF", +] + + +class Marker: + def __init__(self, desc: str): + self.desc = desc + + def __repr__(self) -> str: + return self.desc + + +# To be used as default value when `None` is not an option. +_DEFAULT_MARKER_: Any = Marker("_DEFAULT_MARKER_") + + +class OmegaConfDumper(yaml.Dumper): # type: ignore + str_representer_added = False + + @staticmethod + def str_representer(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode: + with_quotes = yaml_is_bool(data) or is_int(data) or is_float(data) + return dumper.represent_scalar( + yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG, + data, + style=("'" if with_quotes else None), + ) + + +def get_omega_conf_dumper() -> Type[OmegaConfDumper]: + if not OmegaConfDumper.str_representer_added: + OmegaConfDumper.add_representer(str, OmegaConfDumper.str_representer) + OmegaConfDumper.str_representer_added = True + return OmegaConfDumper + + +def yaml_is_bool(b: str) -> bool: + return b in YAML_BOOL_TYPES + + +def get_yaml_loader() -> Any: + class OmegaConfLoader(yaml.SafeLoader): # type: ignore + def construct_mapping(self, node: yaml.Node, deep: bool = False) -> Any: + keys = set() + for key_node, value_node in node.value: + if key_node.tag != yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG: + continue + if key_node.value in keys: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", + node.start_mark, + f"found duplicate key {key_node.value}", + key_node.start_mark, + ) + keys.add(key_node.value) + return super().construct_mapping(node, deep=deep) + + loader = OmegaConfLoader + loader.add_implicit_resolver( + "tag:yaml.org,2002:float", + re.compile( + """^(?: + [-+]?[0-9]+(?:_[0-9]+)*\\.[0-9_]*(?:[eE][-+]?[0-9]+)? + |[-+]?[0-9]+(?:_[0-9]+)*(?:[eE][-+]?[0-9]+) + |\\.[0-9]+(?:_[0-9]+)*(?:[eE][-+][0-9]+)? + |[-+]?[0-9]+(?:_[0-9]+)*(?::[0-5]?[0-9])+\\.[0-9_]* + |[-+]?\\.(?:inf|Inf|INF) + |\\.(?:nan|NaN|NAN))$""", + re.X, + ), + list("-+0123456789."), + ) + loader.yaml_implicit_resolvers = { + key: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:timestamp"] + for key, resolvers in loader.yaml_implicit_resolvers.items() + } + + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.Path", + lambda loader, node: pathlib.Path(*loader.construct_sequence(node)), + ) + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.PosixPath", + lambda loader, node: pathlib.PosixPath(*loader.construct_sequence(node)), + ) + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.WindowsPath", + lambda loader, node: pathlib.WindowsPath(*loader.construct_sequence(node)), + ) + + return loader + + +def _get_class(path: str) -> type: + from importlib import import_module + + module_path, _, class_name = path.rpartition(".") + mod = import_module(module_path) + try: + klass: type = getattr(mod, class_name) + except AttributeError: + raise ImportError(f"Class {class_name} is not in module {module_path}") + return klass + + +def is_union_annotation(type_: Any) -> bool: + if sys.version_info >= (3, 10): # pragma: no cover + if isinstance(type_, types.UnionType): + return True + return getattr(type_, "__origin__", None) is Union + + +def _resolve_optional(type_: Any) -> Tuple[bool, Any]: + """Check whether `type_` is equivalent to `typing.Optional[T]` for some T.""" + if is_union_annotation(type_): + args = type_.__args__ + if NoneType in args: + optional = True + args = tuple(a for a in args if a is not NoneType) + else: + optional = False + if len(args) == 1: + return optional, args[0] + elif len(args) >= 2: + return optional, Union[args] + else: + assert False + + if type_ is Any: + return True, Any + + if type_ in (None, NoneType): + return True, NoneType + + return False, type_ + + +def _is_optional(obj: Any, key: Optional[Union[int, str]] = None) -> bool: + """Check `obj` metadata to see if the given node is optional.""" + from .base import Container, Node + + if key is not None: + assert isinstance(obj, Container) + obj = obj._get_node(key) + assert isinstance(obj, Node) + return obj._is_optional() + + +def _resolve_forward(type_: Type[Any], module: str) -> Type[Any]: + import typing # lgtm [py/import-and-import-from] + + forward = typing.ForwardRef if hasattr(typing, "ForwardRef") else typing._ForwardRef # type: ignore + if type(type_) is forward: + return _get_class(f"{module}.{type_.__forward_arg__}") + else: + if is_dict_annotation(type_): + kt, vt = get_dict_key_value_types(type_) + if kt is not None: + kt = _resolve_forward(kt, module=module) + if vt is not None: + vt = _resolve_forward(vt, module=module) + return Dict[kt, vt] # type: ignore + if is_list_annotation(type_): + et = get_list_element_type(type_) + if et is not None: + et = _resolve_forward(et, module=module) + return List[et] # type: ignore + if is_tuple_annotation(type_): + its = get_tuple_item_types(type_) + its = tuple(_resolve_forward(it, module=module) for it in its) + return Tuple[its] # type: ignore + + return type_ + + +def extract_dict_subclass_data(obj: Any, parent: Any) -> Optional[Dict[str, Any]]: + """Check if obj is an instance of a subclass of Dict. If so, extract the Dict keys/values.""" + from omegaconf.omegaconf import _maybe_wrap + + is_type = isinstance(obj, type) + obj_type = obj if is_type else type(obj) + subclasses_dict = is_dict_subclass(obj_type) + + if subclasses_dict: + warnings.warn( + f"Class `{obj_type.__name__}` subclasses `Dict`." + + " Subclassing `Dict` in Structured Config classes is deprecated," + + " see github.com/omry/omegaconf/issues/663", + UserWarning, + stacklevel=9, + ) + + if is_type: + return None + elif subclasses_dict: + dict_subclass_data = {} + key_type, element_type = get_dict_key_value_types(obj_type) + for name, value in obj.items(): + is_optional, type_ = _resolve_optional(element_type) + type_ = _resolve_forward(type_, obj.__module__) + try: + dict_subclass_data[name] = _maybe_wrap( + ref_type=type_, + is_optional=is_optional, + key=name, + value=value, + parent=parent, + ) + except ValidationError as ex: + format_and_raise(node=None, key=name, value=value, cause=ex, msg=str(ex)) + return dict_subclass_data + else: + return None + + +def get_attr_class_fields(obj: Any) -> List["attr.Attribute[Any]"]: + is_type = isinstance(obj, type) + obj_type = obj if is_type else type(obj) + fields = attr.fields_dict(obj_type).values() + return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True] + + +def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: + from omegaconf.omegaconf import OmegaConf, _maybe_wrap + + flags = {"allow_objects": allow_objects} if allow_objects is not None else {} + + from omegaconf import MISSING + + d = {} + is_type = isinstance(obj, type) + obj_type = obj if is_type else type(obj) + dummy_parent = OmegaConf.create({}, flags=flags) + dummy_parent._metadata.object_type = obj_type + resolved_hints = get_type_hints(obj_type) + + for attrib in get_attr_class_fields(obj): + name = attrib.name + is_optional, type_ = _resolve_optional(resolved_hints[name]) + type_ = _resolve_forward(type_, obj.__module__) + if not is_type: + value = getattr(obj, name) + else: + value = attrib.default + if value == attr.NOTHING: + value = MISSING + if is_union_annotation(type_) and not is_supported_union_annotation(type_): + e = ConfigValueError(f"Unions of containers are not supported:\n{name}: {type_str(type_)}") + format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) + + try: + d[name] = _maybe_wrap( + ref_type=type_, + is_optional=is_optional, + key=name, + value=value, + parent=dummy_parent, + ) + except (ValidationError, GrammarParseError) as ex: + format_and_raise(node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)) + d[name]._set_parent(None) + dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent) + if dict_subclass_data is not None: + d.update(dict_subclass_data) + return d + + +def get_dataclass_fields(obj: Any) -> List["dataclasses.Field[Any]"]: + fields = dataclasses.fields(obj) + return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True] + + +def get_dataclass_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: + from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap + + flags = {"allow_objects": allow_objects} if allow_objects is not None else {} + d = {} + is_type = isinstance(obj, type) + obj_type = get_type_of(obj) + dummy_parent = OmegaConf.create({}, flags=flags) + dummy_parent._metadata.object_type = obj_type + resolved_hints = get_type_hints(obj_type) + for field in get_dataclass_fields(obj): + name = field.name + is_optional, type_ = _resolve_optional(resolved_hints[field.name]) + type_ = _resolve_forward(type_, obj.__module__) + has_default = field.default != dataclasses.MISSING + has_default_factory = field.default_factory != dataclasses.MISSING + + if not is_type: + value = getattr(obj, name) + else: + if has_default: + value = field.default + elif has_default_factory: + value = field.default_factory() # type: ignore + else: + value = MISSING + + if is_union_annotation(type_) and not is_supported_union_annotation(type_): + e = ConfigValueError(f"Unions of containers are not supported:\n{name}: {type_str(type_)}") + format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) + try: + d[name] = _maybe_wrap( + ref_type=type_, + is_optional=is_optional, + key=name, + value=value, + parent=dummy_parent, + ) + except (ValidationError, GrammarParseError) as ex: + format_and_raise(node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)) + d[name]._set_parent(None) + dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent) + if dict_subclass_data is not None: + d.update(dict_subclass_data) + return d + + +def is_dataclass(obj: Any) -> bool: + from omegaconf.base import Node + + if dataclasses is None or isinstance(obj, Node): + return False + is_dataclass = dataclasses.is_dataclass(obj) + assert isinstance(is_dataclass, bool) + return is_dataclass + + +def is_attr_class(obj: Any) -> bool: + from omegaconf.base import Node + + if attr is None or isinstance(obj, Node): + return False + return attr.has(obj) + + +def is_structured_config(obj: Any) -> bool: + return is_attr_class(obj) or is_dataclass(obj) + + +def is_dataclass_frozen(type_: Any) -> bool: + return type_.__dataclass_params__.frozen # type: ignore + + +def is_attr_frozen(type_: type) -> bool: + # This is very hacky and probably fragile as well. + # Unfortunately currently there isn't an official API in attr that can detect that. + # noinspection PyProtectedMember + return type_.__setattr__ == attr._make._frozen_setattrs # type: ignore + + +def get_type_of(class_or_object: Any) -> Type[Any]: + type_ = class_or_object + if not isinstance(type_, type): + type_ = type(class_or_object) + assert isinstance(type_, type) + return type_ + + +def is_structured_config_frozen(obj: Any) -> bool: + type_ = get_type_of(obj) + + if is_dataclass(type_): + return is_dataclass_frozen(type_) + if is_attr_class(type_): + return is_attr_frozen(type_) + return False + + +def get_structured_config_init_field_names(obj: Any) -> List[str]: + fields: Union[List["dataclasses.Field[Any]"], List["attr.Attribute[Any]"]] + if is_dataclass(obj): + fields = get_dataclass_fields(obj) + elif is_attr_class(obj): + fields = get_attr_class_fields(obj) + else: + raise ValueError(f"Unsupported type: {type(obj).__name__}") + return [f.name for f in fields if f.init] + + +def get_structured_config_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: + if is_dataclass(obj): + return get_dataclass_data(obj, allow_objects=allow_objects) + elif is_attr_class(obj): + return get_attr_data(obj, allow_objects=allow_objects) + else: + raise ValueError(f"Unsupported type: {type(obj).__name__}") + + +class ValueKind(Enum): + VALUE = 0 + MANDATORY_MISSING = 1 + INTERPOLATION = 2 + + +def _is_missing_value(value: Any) -> bool: + from omegaconf import Node + + if isinstance(value, Node): + value = value._value() + return _is_missing_literal(value) + + +def _is_missing_literal(value: Any) -> bool: + # Uses literal '???' instead of the MISSING const for performance reasons. + return isinstance(value, str) and value == "???" + + +def _is_none(value: Any, resolve: bool = False, throw_on_resolution_failure: bool = True) -> bool: + from omegaconf import Node + + if not isinstance(value, Node): + return value is None + + if resolve: + value = value._maybe_dereference_node(throw_on_resolution_failure=throw_on_resolution_failure) + if not throw_on_resolution_failure and value is None: + # Resolution failure: consider that it is *not* None. + return False + assert isinstance(value, Node) + + return value._is_none() + + +def get_value_kind(value: Any, strict_interpolation_validation: bool = False) -> ValueKind: + """ + Determine the kind of a value + Examples: + VALUE: "10", "20", True + MANDATORY_MISSING: "???" + INTERPOLATION: "${foo.bar}", "${foo.${bar}}", "${foo:bar}", "[${foo}, ${bar}]", + "ftp://${host}/path", "${foo:${bar}, [true], {'baz': ${baz}}}" + + :param value: Input to classify. + :param strict_interpolation_validation: If `True`, then when `value` is a string + containing "${", it is parsed to validate the interpolation syntax. If `False`, + this parsing step is skipped: this is more efficient, but will not detect errors. + """ + + if _is_missing_value(value): + return ValueKind.MANDATORY_MISSING + + if _is_interpolation(value, strict_interpolation_validation): + return ValueKind.INTERPOLATION + + return ValueKind.VALUE + + +def _is_interpolation(v: Any, strict_interpolation_validation: bool = False) -> bool: + from omegaconf import Node + + if isinstance(v, Node): + v = v._value() + + if isinstance(v, str) and _is_interpolation_string(v, strict_interpolation_validation): + return True + return False + + +def _is_interpolation_string(value: str, strict_interpolation_validation: bool) -> bool: + # We identify potential interpolations by the presence of "${" in the string. + # Note that escaped interpolations (ex: "esc: \${bar}") are identified as + # interpolations: this is intended, since they must be processed as interpolations + # for the string to be properly un-escaped. + # Keep in mind that invalid interpolations will only be detected when + # `strict_interpolation_validation` is True. + if "${" in value: + if strict_interpolation_validation: + # First try the cheap regex matching that detects common interpolations. + if SIMPLE_INTERPOLATION_PATTERN.match(value) is None: + # If no match, do the more expensive grammar parsing to detect errors. + parse(value) + return True + return False + + +def _is_special(value: Any) -> bool: + """Special values are None, MISSING, and interpolation.""" + return _is_none(value) or get_value_kind(value) in ( + ValueKind.MANDATORY_MISSING, + ValueKind.INTERPOLATION, + ) + + +def is_float(st: str) -> bool: + try: + float(st) + return True + except ValueError: + return False + + +def is_int(st: str) -> bool: + try: + int(st) + return True + except ValueError: + return False + + +def is_primitive_list(obj: Any) -> bool: + return isinstance(obj, (list, tuple)) + + +def is_primitive_dict(obj: Any) -> bool: + t = get_type_of(obj) + return t is dict + + +def is_dict_annotation(type_: Any) -> bool: + if type_ in (dict, Dict): + return True + origin = getattr(type_, "__origin__", None) + # type_dict is a bit hard to detect. + # this support is tentative, if it eventually causes issues in other areas it may be dropped. + if sys.version_info < (3, 7, 0): # pragma: no cover + typed_dict = hasattr(type_, "__base__") and type_.__base__ == Dict + return origin is Dict or type_ is Dict or typed_dict + else: # pragma: no cover + typed_dict = hasattr(type_, "__base__") and type_.__base__ == dict + return origin is dict or typed_dict + + +def is_list_annotation(type_: Any) -> bool: + if type_ in (list, List): + return True + origin = getattr(type_, "__origin__", None) + if sys.version_info < (3, 7, 0): + return origin is List or type_ is List # pragma: no cover + else: + return origin is list # pragma: no cover + + +def is_tuple_annotation(type_: Any) -> bool: + if type_ in (tuple, Tuple): + return True + origin = getattr(type_, "__origin__", None) + if sys.version_info < (3, 7, 0): + return origin is Tuple or type_ is Tuple # pragma: no cover + else: + return origin is tuple # pragma: no cover + + +def is_supported_union_annotation(obj: Any) -> bool: + """Currently only primitive types are supported in Unions, e.g. Union[int, str]""" + if not is_union_annotation(obj): + return False + args = obj.__args__ + return all(is_primitive_type_annotation(arg) for arg in args) + + +def is_dict_subclass(type_: Any) -> bool: + return type_ is not None and isinstance(type_, type) and issubclass(type_, Dict) + + +def is_dict(obj: Any) -> bool: + return is_primitive_dict(obj) or is_dict_annotation(obj) or is_dict_subclass(obj) + + +def is_primitive_container(obj: Any) -> bool: + return is_primitive_list(obj) or is_primitive_dict(obj) + + +def get_list_element_type(ref_type: Optional[Type[Any]]) -> Any: + args = getattr(ref_type, "__args__", None) + if ref_type is not List and args is not None and args[0]: + element_type = args[0] + else: + element_type = Any + return element_type + + +def get_tuple_item_types(ref_type: Type[Any]) -> Tuple[Any, ...]: + args = getattr(ref_type, "__args__", None) + if args in (None, ()): + args = (Any, ...) + assert isinstance(args, tuple) + return args + + +def get_dict_key_value_types(ref_type: Any) -> Tuple[Any, Any]: + args = getattr(ref_type, "__args__", None) + if args is None: + bases = getattr(ref_type, "__orig_bases__", None) + if bases is not None and len(bases) > 0: + args = getattr(bases[0], "__args__", None) + + key_type: Any + element_type: Any + if ref_type is None or ref_type == Dict: + key_type = Any + element_type = Any + else: + if args is not None: + key_type = args[0] + element_type = args[1] + else: + key_type = Any + element_type = Any + + return key_type, element_type + + +def is_valid_value_annotation(type_: Any) -> bool: + _, type_ = _resolve_optional(type_) + return ( + type_ is Any + or is_primitive_type_annotation(type_) + or is_structured_config(type_) + or is_container_annotation(type_) + or is_supported_union_annotation(type_) + ) + + +def _valid_dict_key_annotation_type(type_: Any) -> bool: + from omegaconf import DictKeyType + + return type_ is None or type_ is Any or issubclass(type_, DictKeyType.__args__) # type: ignore + + +def is_primitive_type_annotation(type_: Any) -> bool: + type_ = get_type_of(type_) + return issubclass(type_, (Enum, pathlib.Path)) or type_ in BUILTIN_VALUE_TYPES + + +def _get_value(value: Any) -> Any: + from .base import Container, UnionNode + from .nodes import ValueNode + + if isinstance(value, ValueNode): + return value._value() + elif isinstance(value, Container): + boxed = value._value() + if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): + return boxed + elif isinstance(value, UnionNode): + boxed = value._value() + if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): + return boxed + else: + return _get_value(boxed) # pass through value of boxed node + + # return primitives and regular OmegaConf Containers as is + return value + + +def get_type_hint(obj: Any, key: Any = None) -> Optional[Type[Any]]: + from omegaconf import Container, Node + + if isinstance(obj, Container): + if key is not None: + obj = obj._get_node(key) + else: + if key is not None: + raise ValueError("Key must only be provided when obj is a container") + + if isinstance(obj, Node): + ref_type = obj._metadata.ref_type + if obj._is_optional() and ref_type is not Any: + return Optional[ref_type] # type: ignore + else: + return ref_type + else: + return Any # type: ignore + + +def _raise(ex: Exception, cause: Exception) -> None: + # Set the environment variable OC_CAUSE=1 to get a stacktrace that includes the + # causing exception. + env_var = os.environ["OC_CAUSE"] if "OC_CAUSE" in os.environ else None + debugging = sys.gettrace() is not None + full_backtrace = (debugging and not env_var == "0") or (env_var == "1") + if full_backtrace: + ex.__cause__ = cause + else: + ex.__cause__ = None + raise ex.with_traceback(sys.exc_info()[2]) # set env var OC_CAUSE=1 for full trace + + +def format_and_raise( + node: Any, + key: Any, + value: Any, + msg: str, + cause: Exception, + type_override: Any = None, +) -> None: + from omegaconf import OmegaConf + from omegaconf.base import Node + + if isinstance(cause, AssertionError): + raise + + if isinstance(cause, OmegaConfBaseException) and cause._initialized: + ex = cause + if type_override is not None: + ex = type_override(str(cause)) + ex.__dict__ = copy.deepcopy(cause.__dict__) + _raise(ex, cause) + + object_type: Optional[Type[Any]] + object_type_str: Optional[str] = None + ref_type: Optional[Type[Any]] + ref_type_str: Optional[str] + + child_node: Optional[Node] = None + if node is None: + full_key = key if key is not None else "" + object_type = None + ref_type = None + ref_type_str = None + else: + if key is not None and not node._is_none(): + child_node = node._get_node(key, validate_access=False) + + try: + full_key = node._get_full_key(key=key) + except Exception as exc: + # Since we are handling an exception, raising a different one here would + # be misleading. Instead, we display it in the key. + full_key = f"" + + object_type = OmegaConf.get_type(node) + object_type_str = type_str(object_type) + + ref_type = get_type_hint(node) + ref_type_str = type_str(ref_type) + + msg = string.Template(msg).safe_substitute( + REF_TYPE=ref_type_str, + OBJECT_TYPE=object_type_str, + KEY=key, + FULL_KEY=full_key, + VALUE=value, + VALUE_TYPE=type_str(type(value), include_module_name=True), + KEY_TYPE=f"{type(key).__name__}", + ) + + if ref_type not in (None, Any): + template = dedent( + """\ + $MSG + full_key: $FULL_KEY + reference_type=$REF_TYPE + object_type=$OBJECT_TYPE""" + ) + else: + template = dedent( + """\ + $MSG + full_key: $FULL_KEY + object_type=$OBJECT_TYPE""" + ) + s = string.Template(template=template) + + message = s.substitute(REF_TYPE=ref_type_str, OBJECT_TYPE=object_type_str, MSG=msg, FULL_KEY=full_key) + exception_type = type(cause) if type_override is None else type_override + if exception_type == TypeError: + exception_type = ConfigTypeError + elif exception_type == IndexError: + exception_type = ConfigIndexError + + ex = exception_type(f"{message}") + if issubclass(exception_type, OmegaConfBaseException): + ex._initialized = True + ex.msg = message + ex.parent_node = node + ex.child_node = child_node + ex.key = key + ex.full_key = full_key + ex.value = value + ex.object_type = object_type + ex.object_type_str = object_type_str + ex.ref_type = ref_type + ex.ref_type_str = ref_type_str + + _raise(ex, cause) + + +def type_str(t: Any, include_module_name: bool = False) -> str: + is_optional, t = _resolve_optional(t) + if t is NoneType: + return str(t.__name__) + if t is Any: + return "Any" + if t is ...: + return "..." + + if hasattr(t, "__name__"): + name = str(t.__name__) + elif getattr(t, "_name", None) is not None: # pragma: no cover + name = str(t._name) + elif getattr(t, "__origin__", None) is not None: # pragma: no cover + name = type_str(t.__origin__) + else: + name = str(t) + if name.startswith("typing."): # pragma: no cover + name = name[len("typing.") :] + + args = getattr(t, "__args__", None) + if args is not None: + args = ", ".join([type_str(t, include_module_name=include_module_name) for t in t.__args__]) + ret = f"{name}[{args}]" + else: + ret = name + if include_module_name: + if ( + hasattr(t, "__module__") + and t.__module__ != "builtins" + and t.__module__ != "typing" + and not t.__module__.startswith("omegaconf.") + ): + module_prefix = str(t.__module__) + "." + else: + module_prefix = "" + ret = module_prefix + ret + if is_optional: + return f"Optional[{ret}]" + else: + return ret + + +def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any: + from omegaconf import OmegaConf + + if is_primitive_container(target): + assert isinstance(target, (list, dict)) + target = OmegaConf.create(target, flags=flags) + elif is_structured_config(target): + target = OmegaConf.structured(target, flags=flags) + elif not OmegaConf.is_config(target): + raise ValueError( + "Invalid input. Supports one of " + + "[dict,list,DictConfig,ListConfig,dataclass,dataclass instance,attr class,attr class instance]" + ) + + return target + + +def is_generic_list(type_: Any) -> bool: + """ + Checks if a type is a generic list, for example: + list returns False + typing.List returns False + typing.List[T] returns True + + :param type_: variable type + :return: bool + """ + return is_list_annotation(type_) and get_list_element_type(type_) is not None + + +def is_generic_dict(type_: Any) -> bool: + """ + Checks if a type is a generic dict, for example: + list returns False + typing.List returns False + typing.List[T] returns True + + :param type_: variable type + :return: bool + """ + return is_dict_annotation(type_) and len(get_dict_key_value_types(type_)) > 0 + + +def is_container_annotation(type_: Any) -> bool: + return is_list_annotation(type_) or is_dict_annotation(type_) + + +def split_key(key: str) -> List[str]: + """ + Split a full key path into its individual components. + + This is similar to `key.split(".")` but also works with the getitem syntax: + "a.b" -> ["a", "b"] + "a[b]" -> ["a", "b"] + ".a.b[c].d" -> ["", "a", "b", "c", "d"] + "[a].b" -> ["a", "b"] + """ + # Obtain the first part of the key (in docstring examples: a, a, .a, '') + first = KEY_PATH_HEAD.match(key) + assert first is not None + first_stop = first.span()[1] + + # `tokens` will contain all elements composing the key. + tokens = key[0:first_stop].split(".") + + # Optimization in case `key` has no other component: we are done. + if first_stop == len(key): + return tokens + + if key[first_stop] == "[" and not tokens[-1]: + # This is a special case where the first key starts with brackets, e.g. + # [a] or ..[a]. In that case there is an extra "" in `tokens` that we + # need to get rid of: + # [a] -> tokens = [""] but we would like [] + # ..[a] -> tokens = ["", "", ""] but we would like ["", ""] + tokens.pop() + + # Identify other key elements (in docstring examples: b, b, b/c/d, b) + others = KEY_PATH_OTHER.findall(key[first_stop:]) + + # There are two groups in the `KEY_PATH_OTHER` regex: one for keys starting + # with a dot (.b, .d) and one for keys starting with a bracket ([b], [c]). + # Only one group can be non-empty. + tokens += [dot_key if dot_key else bracket_key for dot_key, bracket_key in others] + + return tokens + + +# Similar to Python 3.7+'s `contextlib.nullcontext` (which should be used instead, +# once support for Python 3.6 is dropped). +@contextmanager +def nullcontext(enter_result: Any = None) -> Iterator[Any]: + yield enter_result diff --git a/omegaconf/base.py b/omegaconf/base.py new file mode 100644 index 000000000..d5eff9a0d --- /dev/null +++ b/omegaconf/base.py @@ -0,0 +1,947 @@ +import copy +import sys +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, Union + +from antlr4 import ParserRuleContext + +from ._utils import ( + _DEFAULT_MARKER_, + NoneType, + ValueKind, + _get_value, + _is_interpolation, + _is_missing_value, + _is_special, + format_and_raise, + get_value_kind, + is_union_annotation, + is_valid_value_annotation, + split_key, + type_str, +) +from .errors import ( + ConfigKeyError, + ConfigTypeError, + InterpolationKeyError, + InterpolationResolutionError, + InterpolationToMissingValueError, + InterpolationValidationError, + MissingMandatoryValue, + UnsupportedInterpolationType, + ValidationError, +) +from .grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser +from .grammar_parser import parse +from .grammar_visitor import GrammarVisitor + +DictKeyType = Union[str, bytes, int, Enum, float, bool] + + +@dataclass +class Metadata: + ref_type: Union[Type[Any], Any] + + object_type: Union[Type[Any], Any] + + optional: bool + + key: Any + + # Flags have 3 modes: + # unset : inherit from parent (None if no parent specifies) + # set to true: flag is true + # set to false: flag is false + flags: Optional[Dict[str, bool]] = None + + # If True, when checking the value of a flag, if the flag is not set None is returned + # otherwise, the parent node is queried. + flags_root: bool = False + + resolver_cache: Dict[str, Any] = field(default_factory=lambda: defaultdict(dict)) + + def __post_init__(self) -> None: + if self.flags is None: + self.flags = {} + + @property + def type_hint(self) -> Union[Type[Any], Any]: + """Compute `type_hint` from `self.optional` and `self.ref_type`""" + # For compatibility with pickled OmegaConf objects created using older + # versions of OmegaConf, we store `ref_type` and `object_type` + # separately (rather than storing `type_hint` directly). + if self.optional: + return Optional[self.ref_type] + else: + return self.ref_type + + +@dataclass +class ContainerMetadata(Metadata): + key_type: Any = None + element_type: Any = None + + def __post_init__(self) -> None: + if self.ref_type is None: + self.ref_type = Any + assert self.key_type is Any or isinstance(self.key_type, type) + if self.element_type is not None: + if not is_valid_value_annotation(self.element_type): + raise ValidationError( + f"Unsupported value type: '{type_str(self.element_type, include_module_name=True)}'" + ) + + if self.flags is None: + self.flags = {} + + +class Node(ABC): + _metadata: Metadata + + _parent: Optional["Box"] + _flags_cache: Optional[Dict[str, Optional[bool]]] + + def __init__(self, parent: Optional["Box"], metadata: Metadata): + self.__dict__["_metadata"] = metadata + self.__dict__["_parent"] = parent + self.__dict__["_flags_cache"] = None + + def __getstate__(self) -> Dict[str, Any]: + # Overridden to ensure that the flags cache is cleared on serialization. + state_dict = copy.copy(self.__dict__) + del state_dict["_flags_cache"] + return state_dict + + def __setstate__(self, state_dict: Dict[str, Any]) -> None: + self.__dict__.update(state_dict) + self.__dict__["_flags_cache"] = None + + def _set_parent(self, parent: Optional["Box"]) -> None: + assert parent is None or isinstance(parent, Box) + self.__dict__["_parent"] = parent + self._invalidate_flags_cache() + + def _invalidate_flags_cache(self) -> None: + self.__dict__["_flags_cache"] = None + + def _get_parent(self) -> Optional["Box"]: + parent = self.__dict__["_parent"] + assert parent is None or isinstance(parent, Box) + return parent + + def _get_parent_container(self) -> Optional["Container"]: + """ + Like _get_parent, but returns the grandparent + in the case where `self` is wrapped by a UnionNode. + """ + parent = self.__dict__["_parent"] + assert parent is None or isinstance(parent, Box) + + if isinstance(parent, UnionNode): + grandparent = parent.__dict__["_parent"] + assert grandparent is None or isinstance(grandparent, Container) + return grandparent + else: + assert parent is None or isinstance(parent, Container) + return parent + + def _set_flag( + self, + flags: Union[List[str], str], + values: Union[List[Optional[bool]], Optional[bool]], + ) -> "Node": + if isinstance(flags, str): + flags = [flags] + + if values is None or isinstance(values, bool): + values = [values] + + if len(values) == 1: + values = len(flags) * values + + if len(flags) != len(values): + raise ValueError("Inconsistent lengths of input flag names and values") + + for idx, flag in enumerate(flags): + value = values[idx] + if value is None: + assert self._metadata.flags is not None + if flag in self._metadata.flags: + del self._metadata.flags[flag] + else: + assert self._metadata.flags is not None + self._metadata.flags[flag] = value + self._invalidate_flags_cache() + return self + + def _get_node_flag(self, flag: str) -> Optional[bool]: + """ + :param flag: flag to inspect + :return: the state of the flag on this node. + """ + assert self._metadata.flags is not None + return self._metadata.flags.get(flag) + + def _get_flag(self, flag: str) -> Optional[bool]: + cache = self.__dict__["_flags_cache"] + if cache is None: + cache = self.__dict__["_flags_cache"] = {} + + ret = cache.get(flag, _DEFAULT_MARKER_) + if ret is _DEFAULT_MARKER_: + ret = self._get_flag_no_cache(flag) + cache[flag] = ret + assert ret is None or isinstance(ret, bool) + return ret + + def _get_flag_no_cache(self, flag: str) -> Optional[bool]: + """ + Returns True if this config node flag is set + A flag is set if node.set_flag(True) was called + or one if it's parents is flag is set + :return: + """ + flags = self._metadata.flags + assert flags is not None + if flag in flags and flags[flag] is not None: + return flags[flag] + + if self._is_flags_root(): + return None + + parent = self._get_parent() + if parent is None: + return None + else: + # noinspection PyProtectedMember + return parent._get_flag(flag) + + def _format_and_raise( + self, + key: Any, + value: Any, + cause: Exception, + msg: Optional[str] = None, + type_override: Any = None, + ) -> None: + format_and_raise( + node=self, + key=key, + value=value, + msg=str(cause) if msg is None else msg, + cause=cause, + type_override=type_override, + ) + assert False + + @abstractmethod + def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: + ... + + def _dereference_node(self) -> "Node": + node = self._dereference_node_impl(throw_on_resolution_failure=True) + assert node is not None + return node + + def _maybe_dereference_node( + self, + throw_on_resolution_failure: bool = False, + memo: Optional[Set[int]] = None, + ) -> Optional["Node"]: + return self._dereference_node_impl( + throw_on_resolution_failure=throw_on_resolution_failure, + memo=memo, + ) + + def _dereference_node_impl( + self, + throw_on_resolution_failure: bool, + memo: Optional[Set[int]] = None, + ) -> Optional["Node"]: + if not self._is_interpolation(): + return self + + parent = self._get_parent_container() + if parent is None: + if throw_on_resolution_failure: + raise InterpolationResolutionError("Cannot resolve interpolation for a node without a parent") + return None + assert parent is not None + key = self._key() + return parent._resolve_interpolation_from_parse_tree( + parent=parent, + key=key, + value=self, + parse_tree=parse(_get_value(self)), + throw_on_resolution_failure=throw_on_resolution_failure, + memo=memo, + ) + + def _get_root(self) -> "Container": + root: Optional[Box] = self._get_parent() + if root is None: + assert isinstance(self, Container) + return self + assert root is not None and isinstance(root, Box) + while root._get_parent() is not None: + root = root._get_parent() + assert root is not None and isinstance(root, Box) + assert root is not None and isinstance(root, Container) + return root + + def _is_missing(self) -> bool: + """ + Check if the node's value is `???` (does *not* resolve interpolations). + """ + return _is_missing_value(self) + + def _is_none(self) -> bool: + """ + Check if the node's value is `None` (does *not* resolve interpolations). + """ + return self._value() is None + + @abstractmethod + def __eq__(self, other: Any) -> bool: + ... + + @abstractmethod + def __ne__(self, other: Any) -> bool: + ... + + @abstractmethod + def __hash__(self) -> int: + ... + + @abstractmethod + def _value(self) -> Any: + ... + + @abstractmethod + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + ... + + @abstractmethod + def _is_optional(self) -> bool: + ... + + @abstractmethod + def _is_interpolation(self) -> bool: + ... + + def _key(self) -> Any: + return self._metadata.key + + def _set_key(self, key: Any) -> None: + self._metadata.key = key + + def _is_flags_root(self) -> bool: + return self._metadata.flags_root + + def _set_flags_root(self, flags_root: bool) -> None: + if self._metadata.flags_root != flags_root: + self._metadata.flags_root = flags_root + self._invalidate_flags_cache() + + def _has_ref_type(self) -> bool: + return self._metadata.ref_type is not Any + + +class Box(Node): + """ + Base class for nodes that can contain other nodes. + Concrete subclasses include DictConfig, ListConfig, and UnionNode. + """ + + _content: Any + + def __init__(self, parent: Optional["Box"], metadata: Metadata): + super().__init__(parent=parent, metadata=metadata) + self.__dict__["_content"] = None + + def __copy__(self) -> Any: + # real shallow copy is impossible because of the reference to the parent. + return copy.deepcopy(self) + + def _re_parent(self) -> None: + from .dictconfig import DictConfig + from .listconfig import ListConfig + + # update parents of first level Config nodes to self + + if isinstance(self, DictConfig): + content = self.__dict__["_content"] + if isinstance(content, dict): + for _key, value in self.__dict__["_content"].items(): + if value is not None: + value._set_parent(self) + if isinstance(value, Box): + value._re_parent() + elif isinstance(self, ListConfig): + content = self.__dict__["_content"] + if isinstance(content, list): + for item in self.__dict__["_content"]: + if item is not None: + item._set_parent(self) + if isinstance(item, Box): + item._re_parent() + elif isinstance(self, UnionNode): + content = self.__dict__["_content"] + if isinstance(content, Node): + content._set_parent(self) + if isinstance(content, Box): # pragma: no cover + # No coverage here as support for containers inside + # UnionNode is not yet implemented + content._re_parent() + + +class Container(Box): + """ + Container tagging interface + """ + + _metadata: ContainerMetadata + + @abstractmethod + def _get_child( + self, + key: Any, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + ... + + @abstractmethod + def _get_node( + self, + key: Any, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + ... + + @abstractmethod + def __delitem__(self, key: Any) -> None: + ... + + @abstractmethod + def __setitem__(self, key: Any, value: Any) -> None: + ... + + @abstractmethod + def __iter__(self) -> Iterator[Any]: + ... + + @abstractmethod + def __getitem__(self, key_or_index: Any) -> Any: + ... + + def _resolve_key_and_root(self, key: str) -> Tuple["Container", str]: + orig = key + if not key.startswith("."): + return self._get_root(), key + else: + root: Optional[Container] = self + assert key.startswith(".") + while True: + assert root is not None + key = key[1:] + if not key.startswith("."): + break + root = root._get_parent_container() + if root is None: + raise ConfigKeyError(f"Error resolving key '{orig}'") + + return root, key + + def _select_impl( + self, + key: str, + throw_on_missing: bool, + throw_on_resolution_failure: bool, + memo: Optional[Set[int]] = None, + ) -> Tuple[Optional["Container"], Optional[str], Optional[Node]]: + """ + Select a value using dot separated key sequence + """ + from .omegaconf import _select_one + + if key == "": + return self, "", self + + split = split_key(key) + root: Optional[Container] = self + for i in range(len(split) - 1): + if root is None: + break + + k = split[i] + ret, _ = _select_one( + c=root, + key=k, + throw_on_missing=throw_on_missing, + throw_on_type_error=throw_on_resolution_failure, + ) + if isinstance(ret, Node): + ret = ret._maybe_dereference_node( + throw_on_resolution_failure=throw_on_resolution_failure, + memo=memo, + ) + + if ret is not None and not isinstance(ret, Container): + parent_key = ".".join(split[0 : i + 1]) + child_key = split[i + 1] + raise ConfigTypeError( + f"Error trying to access {key}: node `{parent_key}` " + f"is not a container and thus cannot contain `{child_key}`" + ) + root = ret + + if root is None: + return None, None, None + + last_key = split[-1] + value, _ = _select_one( + c=root, + key=last_key, + throw_on_missing=throw_on_missing, + throw_on_type_error=throw_on_resolution_failure, + ) + if value is None: + return root, last_key, None + + if memo is not None: + vid = id(value) + if vid in memo: + raise InterpolationResolutionError("Recursive interpolation detected") + # push to memo "stack" + memo.add(vid) + + try: + value = root._maybe_resolve_interpolation( + parent=root, + key=last_key, + value=value, + throw_on_resolution_failure=throw_on_resolution_failure, + memo=memo, + ) + finally: + if memo is not None: + # pop from memo "stack" + memo.remove(vid) + + return root, last_key, value + + def _resolve_interpolation_from_parse_tree( + self, + parent: Optional["Container"], + value: "Node", + key: Any, + parse_tree: OmegaConfGrammarParser.ConfigValueContext, + throw_on_resolution_failure: bool, + memo: Optional[Set[int]], + ) -> Optional["Node"]: + """ + Resolve an interpolation. + + This happens in two steps: + 1. The parse tree is visited, which outputs either a `Node` (e.g., + for node interpolations "${foo}"), a string (e.g., for string + interpolations "hello ${name}", or any other arbitrary value + (e.g., or custom interpolations "${foo:bar}"). + 2. This output is potentially validated and converted when the node + being resolved (`value`) is typed. + + If an error occurs in one of the above steps, an `InterpolationResolutionError` + (or a subclass of it) is raised, *unless* `throw_on_resolution_failure` is set + to `False` (in which case the return value is `None`). + + :param parent: Parent of the node being resolved. + :param value: Node being resolved. + :param key: The associated key in the parent. + :param parse_tree: The parse tree as obtained from `grammar_parser.parse()`. + :param throw_on_resolution_failure: If `False`, then exceptions raised during + the resolution of the interpolation are silenced, and instead `None` is + returned. + + :return: A `Node` that contains the interpolation result. This may be an existing + node in the config (in the case of a node interpolation "${foo}"), or a new + node that is created to wrap the interpolated value. It is `None` if and only if + `throw_on_resolution_failure` is `False` and an error occurs during resolution. + """ + + try: + resolved = self.resolve_parse_tree(parse_tree=parse_tree, node=value, key=key, memo=memo) + except InterpolationResolutionError: + if throw_on_resolution_failure: + raise + return None + + return self._validate_and_convert_interpolation_result( + parent=parent, + value=value, + key=key, + resolved=resolved, + throw_on_resolution_failure=throw_on_resolution_failure, + ) + + def _validate_and_convert_interpolation_result( + self, + parent: Optional["Container"], + value: "Node", + key: Any, + resolved: Any, + throw_on_resolution_failure: bool, + ) -> Optional["Node"]: + from .nodes import AnyNode, InterpolationResultNode, ValueNode + + # If the output is not a Node already (e.g., because it is the output of a + # custom resolver), then we will need to wrap it within a Node. + must_wrap = not isinstance(resolved, Node) + + # If the node is typed, validate (and possibly convert) the result. + if isinstance(value, ValueNode) and not isinstance(value, AnyNode): + res_value = _get_value(resolved) + try: + conv_value = value.validate_and_convert(res_value) + except ValidationError as e: + if throw_on_resolution_failure: + self._format_and_raise( + key=key, + value=res_value, + cause=e, + msg=f"While dereferencing interpolation '{value}': {e}", + type_override=InterpolationValidationError, + ) + return None + + # If the converted value is of the same type, it means that no conversion + # was actually needed. As a result, we can keep the original `resolved` + # (and otherwise, the converted value must be wrapped into a new node). + if type(conv_value) != type(res_value): + must_wrap = True + resolved = conv_value + + if must_wrap: + return InterpolationResultNode(value=resolved, key=key, parent=parent) + else: + assert isinstance(resolved, Node) + return resolved + + def _validate_not_dereferencing_to_parent(self, node: Node, target: Node) -> None: + parent: Optional[Node] = node + while parent is not None: + if parent is target: + raise InterpolationResolutionError("Interpolation to parent node detected") + parent = parent._get_parent() + + def _resolve_node_interpolation(self, inter_key: str, memo: Optional[Set[int]]) -> "Node": + """A node interpolation is of the form `${foo.bar}`""" + try: + root_node, inter_key = self._resolve_key_and_root(inter_key) + except ConfigKeyError as exc: + raise InterpolationKeyError( + f"ConfigKeyError while resolving interpolation: {exc}" + ).with_traceback(sys.exc_info()[2]) + + try: + parent, last_key, value = root_node._select_impl( + inter_key, + throw_on_missing=True, + throw_on_resolution_failure=True, + memo=memo, + ) + except MissingMandatoryValue as exc: + raise InterpolationToMissingValueError( + f"MissingMandatoryValue while resolving interpolation: {exc}" + ).with_traceback(sys.exc_info()[2]) + + if parent is None or value is None: + raise InterpolationKeyError(f"Interpolation key '{inter_key}' not found") + else: + self._validate_not_dereferencing_to_parent(node=self, target=value) + return value + + def _evaluate_custom_resolver( + self, + key: Any, + node: Node, + inter_type: str, + inter_args: Tuple[Any, ...], + inter_args_str: Tuple[str, ...], + ) -> Any: + from omegaconf import OmegaConf + + resolver = OmegaConf._get_resolver(inter_type) + if resolver is not None: + root_node = self._get_root() + return resolver( + root_node, + self, + node, + inter_args, + inter_args_str, + ) + else: + raise UnsupportedInterpolationType(f"Unsupported interpolation type {inter_type}") + + def _maybe_resolve_interpolation( + self, + parent: Optional["Container"], + key: Any, + value: Node, + throw_on_resolution_failure: bool, + memo: Optional[Set[int]] = None, + ) -> Optional[Node]: + value_kind = get_value_kind(value) + if value_kind != ValueKind.INTERPOLATION: + return value + + parse_tree = parse(_get_value(value)) + return self._resolve_interpolation_from_parse_tree( + parent=parent, + value=value, + key=key, + parse_tree=parse_tree, + throw_on_resolution_failure=throw_on_resolution_failure, + memo=memo if memo is not None else set(), + ) + + def resolve_parse_tree( + self, + parse_tree: ParserRuleContext, + node: Node, + memo: Optional[Set[int]] = None, + key: Optional[Any] = None, + ) -> Any: + """ + Resolve a given parse tree into its value. + + We make no assumption here on the type of the tree's root, so that the + return value may be of any type. + """ + + def node_interpolation_callback(inter_key: str, memo: Optional[Set[int]]) -> Optional["Node"]: + return self._resolve_node_interpolation(inter_key=inter_key, memo=memo) + + def resolver_interpolation_callback( + name: str, args: Tuple[Any, ...], args_str: Tuple[str, ...] + ) -> Any: + return self._evaluate_custom_resolver( + key=key, + node=node, + inter_type=name, + inter_args=args, + inter_args_str=args_str, + ) + + visitor = GrammarVisitor( + node_interpolation_callback=node_interpolation_callback, + resolver_interpolation_callback=resolver_interpolation_callback, + memo=memo, + ) + try: + return visitor.visit(parse_tree) + except InterpolationResolutionError: + raise + except Exception as exc: + # Other kinds of exceptions are wrapped in an `InterpolationResolutionError`. + raise InterpolationResolutionError( + f"{type(exc).__name__} raised while resolving interpolation: {exc}" + ).with_traceback(sys.exc_info()[2]) + + def _invalidate_flags_cache(self) -> None: + from .dictconfig import DictConfig + from .listconfig import ListConfig + + # invalidate subtree cache only if the cache is initialized in this node. + + if self.__dict__["_flags_cache"] is not None: + self.__dict__["_flags_cache"] = None + if isinstance(self, DictConfig): + content = self.__dict__["_content"] + if isinstance(content, dict): + for value in self.__dict__["_content"].values(): + value._invalidate_flags_cache() + elif isinstance(self, ListConfig): + content = self.__dict__["_content"] + if isinstance(content, list): + for item in self.__dict__["_content"]: + item._invalidate_flags_cache() + + +class SCMode(Enum): + DICT = 1 # Convert to plain dict + DICT_CONFIG = 2 # Keep as OmegaConf DictConfig + INSTANTIATE = 3 # Create a dataclass or attrs class instance + + +class UnionNode(Box): + """ + This class handles Union type hints. The `_content` attribute is either a + child node that is compatible with the given Union ref_type, or it is a + special value (None or MISSING or interpolation). + + Much of the logic for e.g. value assignment and type validation is + delegated to the child node. As such, UnionNode functions as a + "pass-through" node. User apps and downstream libraries should not need to + know about UnionNode (assuming they only use OmegaConf's public API). + """ + + _parent: Optional[Container] + _content: Union[Node, None, str] + + def __init__( + self, + content: Any, + ref_type: Any, + is_optional: bool = True, + key: Any = None, + parent: Optional[Box] = None, + ) -> None: + try: + if not is_union_annotation(ref_type): # pragma: no cover + msg = ( + f"UnionNode got unexpected ref_type {ref_type}. Please file a bug" + + " report at https://github.com/omry/omegaconf/issues" + ) + raise AssertionError(msg) + if not isinstance(parent, (Container, NoneType)): + raise ConfigTypeError("Parent type is not omegaconf.Container") + super().__init__( + parent=parent, + metadata=Metadata( + ref_type=ref_type, + object_type=None, + optional=is_optional, + key=key, + flags={"convert": False}, + ), + ) + self._set_value(content) + except Exception as ex: + format_and_raise(node=None, key=key, value=content, msg=str(ex), cause=ex) + + def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: + parent = self._get_parent() + if parent is None: + if self._metadata.key is None: + return "" + else: + return str(self._metadata.key) + else: + return parent._get_full_key(self._metadata.key) + + def __eq__(self, other: Any) -> bool: + content = self.__dict__["_content"] + if isinstance(content, Node): + ret = content.__eq__(other) + elif isinstance(other, Node): + ret = other.__eq__(content) + else: + ret = content.__eq__(other) + assert isinstance(ret, (bool, type(NotImplemented))) + return ret + + def __ne__(self, other: Any) -> bool: + x = self.__eq__(other) + if x is NotImplemented: + return NotImplemented + return not x + + def __hash__(self) -> int: + return hash(self.__dict__["_content"]) + + def _value(self) -> Union[Node, None, str]: + content = self.__dict__["_content"] + assert isinstance(content, (Node, NoneType, str)) + return content + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + previous_content = self.__dict__["_content"] + previous_metadata = self.__dict__["_metadata"] + try: + self._set_value_impl(value, flags) + except Exception as e: + self.__dict__["_content"] = previous_content + self.__dict__["_metadata"] = previous_metadata + raise e + + def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + from omegaconf.omegaconf import _node_wrap + + ref_type = self._metadata.ref_type + type_hint = self._metadata.type_hint + + value = _get_value(value) + if _is_special(value): + assert isinstance(value, (str, NoneType)) + if value is None: + if not self._is_optional(): + raise ValidationError( + f"Value '$VALUE' is incompatible with type hint '{type_str(type_hint)}'" + ) + self.__dict__["_content"] = value + elif isinstance(value, Container): + raise ValidationError( + f"Cannot assign container '$VALUE' of type '$VALUE_TYPE' to {type_str(type_hint)}" + ) + else: + for candidate_ref_type in ref_type.__args__: + try: + self.__dict__["_content"] = _node_wrap( + value=value, + ref_type=candidate_ref_type, + is_optional=False, + key=None, + parent=self, + ) + break + except ValidationError: + continue + else: + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_str(type_hint)}'" + ) + + def _is_optional(self) -> bool: + return self.__dict__["_metadata"].optional is True + + def _is_interpolation(self) -> bool: + return _is_interpolation(self.__dict__["_content"]) + + def __str__(self) -> str: + return str(self.__dict__["_content"]) + + def __repr__(self) -> str: + return repr(self.__dict__["_content"]) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "UnionNode": + res = object.__new__(type(self)) + for key, value in self.__dict__.items(): + if key not in ("_content", "_parent"): + res.__dict__[key] = copy.deepcopy(value, memo=memo) + + src_content = self.__dict__["_content"] + if isinstance(src_content, Node): + old_parent = src_content.__dict__["_parent"] + try: + src_content.__dict__["_parent"] = None + content_copy = copy.deepcopy(src_content, memo=memo) + content_copy.__dict__["_parent"] = res + finally: + src_content.__dict__["_parent"] = old_parent + else: + # None and strings can be assigned as is + content_copy = src_content + + res.__dict__["_content"] = content_copy + res.__dict__["_parent"] = self.__dict__["_parent"] + return res diff --git a/omegaconf/basecontainer.py b/omegaconf/basecontainer.py new file mode 100644 index 000000000..c22cbfd1d --- /dev/null +++ b/omegaconf/basecontainer.py @@ -0,0 +1,916 @@ +import copy +import sys +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union + +import yaml + +from ._utils import ( + _DEFAULT_MARKER_, + ValueKind, + _ensure_container, + _get_value, + _is_interpolation, + _is_missing_value, + _is_none, + _is_special, + _resolve_optional, + get_structured_config_data, + get_type_hint, + get_value_kind, + get_yaml_loader, + is_container_annotation, + is_dict_annotation, + is_list_annotation, + is_primitive_dict, + is_primitive_type_annotation, + is_structured_config, + is_tuple_annotation, + is_union_annotation, +) +from .base import ( + Box, + Container, + ContainerMetadata, + DictKeyType, + Node, + SCMode, + UnionNode, +) +from .errors import ( + ConfigCycleDetectedException, + ConfigTypeError, + InterpolationResolutionError, + KeyValidationError, + MissingMandatoryValue, + OmegaConfBaseException, + ReadonlyConfigError, + ValidationError, +) + +if TYPE_CHECKING: + from .dictconfig import DictConfig # pragma: no cover + + +class BaseContainer(Container, ABC): + _resolvers: ClassVar[Dict[str, Any]] = {} + + def __init__(self, parent: Optional[Box], metadata: ContainerMetadata): + if not (parent is None or isinstance(parent, Box)): + raise ConfigTypeError("Parent type is not omegaconf.Box") + super().__init__(parent=parent, metadata=metadata) + + def _get_child( + self, + key: Any, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + """Like _get_node, passing through to the nearest concrete Node.""" + child = self._get_node( + key=key, + validate_access=validate_access, + validate_key=validate_key, + throw_on_missing_value=throw_on_missing_value, + throw_on_missing_key=throw_on_missing_key, + ) + if isinstance(child, UnionNode) and not _is_special(child): + value = child._value() + assert isinstance(value, Node) and not isinstance(value, UnionNode) + child = value + return child + + def _resolve_with_default( + self, + key: Union[DictKeyType, int], + value: Node, + default_value: Any = _DEFAULT_MARKER_, + ) -> Any: + """returns the value with the specified key, like obj.key and obj['key']""" + if _is_missing_value(value): + if default_value is not _DEFAULT_MARKER_: + return default_value + raise MissingMandatoryValue("Missing mandatory value: $FULL_KEY") + + resolved_node = self._maybe_resolve_interpolation( + parent=self, + key=key, + value=value, + throw_on_resolution_failure=True, + ) + + return _get_value(resolved_node) + + def __str__(self) -> str: + return self.__repr__() + + def __repr__(self) -> str: + if self.__dict__["_content"] is None: + return "None" + elif self._is_interpolation() or self._is_missing(): + v = self.__dict__["_content"] + return f"'{v}'" + else: + return self.__dict__["_content"].__repr__() # type: ignore + + # Support pickle + def __getstate__(self) -> Dict[str, Any]: + dict_copy = copy.copy(self.__dict__) + + # no need to serialize the flags cache, it can be re-constructed later + dict_copy.pop("_flags_cache", None) + + dict_copy["_metadata"] = copy.copy(dict_copy["_metadata"]) + ref_type = self._metadata.ref_type + if is_container_annotation(ref_type): + if is_dict_annotation(ref_type): + dict_copy["_metadata"].ref_type = Dict + elif is_list_annotation(ref_type): + dict_copy["_metadata"].ref_type = List + else: + assert False + if sys.version_info < (3, 7): # pragma: no cover + element_type = self._metadata.element_type + if is_union_annotation(element_type): + raise OmegaConfBaseException( + "Serializing structured configs with `Union` element type requires python >= 3.7" + ) + return dict_copy + + # Support pickle + def __setstate__(self, d: Dict[str, Any]) -> None: + from omegaconf import DictConfig + from omegaconf._utils import is_generic_dict, is_generic_list + + if isinstance(self, DictConfig): + key_type = d["_metadata"].key_type + + # backward compatibility to load OmegaConf 2.0 configs + if key_type is None: + key_type = Any + d["_metadata"].key_type = key_type + + element_type = d["_metadata"].element_type + + # backward compatibility to load OmegaConf 2.0 configs + if element_type is None: + element_type = Any + d["_metadata"].element_type = element_type + + ref_type = d["_metadata"].ref_type + if is_container_annotation(ref_type): + if is_generic_dict(ref_type): + d["_metadata"].ref_type = Dict[key_type, element_type] # type: ignore + elif is_generic_list(ref_type): + d["_metadata"].ref_type = List[element_type] # type: ignore + else: + assert False + + d["_flags_cache"] = None + self.__dict__.update(d) + + @abstractmethod + def __delitem__(self, key: Any) -> None: + ... + + def __len__(self) -> int: + if self._is_none() or self._is_missing() or self._is_interpolation(): + return 0 + content = self.__dict__["_content"] + return len(content) + + def merge_with_cli(self) -> None: + args_list = sys.argv[1:] + self.merge_with_dotlist(args_list) + + def merge_with_dotlist(self, dotlist: List[str]) -> None: + from omegaconf import OmegaConf + + def fail() -> None: + raise ValueError("Input list must be a list or a tuple of strings") + + if not isinstance(dotlist, (list, tuple)): + fail() + + for arg in dotlist: + if not isinstance(arg, str): + fail() + + idx = arg.find("=") + if idx == -1: + key = arg + value = None + else: + key = arg[0:idx] + value = arg[idx + 1 :] + value = yaml.load(value, Loader=get_yaml_loader()) + + OmegaConf.update(self, key, value) + + def is_empty(self) -> bool: + """return true if config is empty""" + return len(self.__dict__["_content"]) == 0 + + @staticmethod + def _to_content( + conf: Container, + resolve: bool, + throw_on_missing: bool, + enum_to_str: bool = False, + structured_config_mode: SCMode = SCMode.DICT, + ) -> Union[None, Any, str, Dict[DictKeyType, Any], List[Any]]: + from omegaconf import MISSING, DictConfig, ListConfig + + def convert(val: Node) -> Any: + value = val._value() + if enum_to_str and isinstance(value, Enum): + value = f"{value.name}" + + return value + + def get_node_value(key: Union[DictKeyType, int]) -> Any: + try: + node = conf._get_child(key, throw_on_missing_value=throw_on_missing) + except MissingMandatoryValue as e: + conf._format_and_raise(key=key, value=None, cause=e) + assert isinstance(node, Node) + if resolve: + try: + node = node._dereference_node() + except InterpolationResolutionError as e: + conf._format_and_raise(key=key, value=None, cause=e) + + if isinstance(node, Container): + value = BaseContainer._to_content( + node, + resolve=resolve, + throw_on_missing=throw_on_missing, + enum_to_str=enum_to_str, + structured_config_mode=structured_config_mode, + ) + else: + value = convert(node) + return value + + if conf._is_none(): + return None + elif conf._is_missing(): + if throw_on_missing: + conf._format_and_raise( + key=None, + value=None, + cause=MissingMandatoryValue("Missing mandatory value"), + ) + else: + return MISSING + elif not resolve and conf._is_interpolation(): + inter = conf._value() + assert isinstance(inter, str) + return inter + + if resolve: + _conf = conf._dereference_node() + assert isinstance(_conf, Container) + conf = _conf + + if isinstance(conf, DictConfig): + if ( + conf._metadata.object_type not in (dict, None) + and structured_config_mode == SCMode.DICT_CONFIG + ): + return conf + if structured_config_mode == SCMode.INSTANTIATE and is_structured_config( + conf._metadata.object_type + ): + return conf._to_object() + + retdict: Dict[DictKeyType, Any] = {} + for key in conf.keys(): + value = get_node_value(key) + if enum_to_str and isinstance(key, Enum): + key = f"{key.name}" + retdict[key] = value + return retdict + elif isinstance(conf, ListConfig): + retlist: List[Any] = [] + for index in range(len(conf)): + item = get_node_value(index) + retlist.append(item) + + return retlist + assert False + + @staticmethod + def _map_merge( + dest: "BaseContainer", + src: "BaseContainer", + extend_lists: bool = False, + remove_duplicates: bool = False, + ) -> None: + """merge src into dest and return a new copy, does not modified input""" + from omegaconf import AnyNode, DictConfig, ValueNode + + assert isinstance(dest, DictConfig) + assert isinstance(src, DictConfig) + src_type = src._metadata.object_type + src_ref_type = get_type_hint(src) + assert src_ref_type is not None + + # If source DictConfig is: + # - None => set the destination DictConfig to None + # - an interpolation => set the destination DictConfig to be the same interpolation + if src._is_none() or src._is_interpolation(): + dest._set_value(src._value()) + _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) + return + + dest._validate_merge(value=src) + + def expand(node: Container) -> None: + rt = node._metadata.ref_type + val: Any + if rt is not Any: + if is_dict_annotation(rt): + val = {} + elif is_list_annotation(rt) or is_tuple_annotation(rt): + val = [] + else: + val = rt + elif isinstance(node, DictConfig): + val = {} + else: + assert False + + node._set_value(val) + + if src._is_missing() and not dest._is_missing() and is_structured_config(src_ref_type): + # Replace `src` with a prototype of its corresponding structured config + # whose fields are all missing (to avoid overwriting fields in `dest`). + assert src_type is None # src missing, so src's object_type should be None + src_type = src_ref_type + src = _create_structured_with_missing_fields(ref_type=src_ref_type, object_type=src_type) + + if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing(): + expand(dest) + + src_items = list(src) if not src._is_missing() else [] + for key in src_items: + src_node = src._get_node(key, validate_access=False) + dest_node = dest._get_node(key, validate_access=False) + assert isinstance(src_node, Node) + assert dest_node is None or isinstance(dest_node, Node) + src_value = _get_value(src_node) + + src_vk = get_value_kind(src_node) + src_node_missing = src_vk is ValueKind.MANDATORY_MISSING + + if isinstance(dest_node, DictConfig): + dest_node._validate_merge(value=src_node) + + if ( + isinstance(dest_node, Container) + and dest_node._is_none() + and not src_node_missing + and not _is_none(src_node, resolve=True) + ): + expand(dest_node) + + if dest_node is not None and dest_node._is_interpolation(): + target_node = dest_node._maybe_dereference_node() + if isinstance(target_node, Container): + dest[key] = target_node + dest_node = dest._get_node(key) + + is_optional, et = _resolve_optional(dest._metadata.element_type) + if dest_node is None and is_structured_config(et) and not src_node_missing: + # merging into a new node. Use element_type as a base + dest[key] = DictConfig(et, parent=dest, ref_type=et, is_optional=is_optional) + dest_node = dest._get_node(key) + + if dest_node is not None: + if isinstance(dest_node, BaseContainer): + if isinstance(src_node, BaseContainer): + dest_node._merge_with( + src_node, + extend_lists=extend_lists, + remove_duplicates=remove_duplicates, + ) + elif not src_node_missing: + dest.__setitem__(key, src_node) + else: + if isinstance(src_node, BaseContainer): + dest.__setitem__(key, src_node) + else: + assert isinstance(dest_node, (ValueNode, UnionNode)) + assert isinstance(src_node, (ValueNode, UnionNode)) + try: + if isinstance(dest_node, AnyNode): + if src_node_missing: + node = copy.copy(src_node) + # if src node is missing, use the value from the dest_node, + # but validate it against the type of the src node before assigment + node._set_value(dest_node._value()) + else: + node = src_node + dest.__setitem__(key, node) + else: + if not src_node_missing: + dest_node._set_value(src_value) + + except (ValidationError, ReadonlyConfigError) as e: + dest._format_and_raise(key=key, value=src_value, cause=e) + else: + from omegaconf import open_dict + + if is_structured_config(src_type): + # verified to be compatible above in _validate_merge + with open_dict(dest): + dest[key] = src._get_node(key) + else: + dest[key] = src._get_node(key) + + _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) + + # explicit flags on the source config are replacing the flag values in the destination + flags = src._metadata.flags + assert flags is not None + for flag, value in flags.items(): + if value is not None: + dest._set_flag(flag, value) + + @staticmethod + def _list_merge(dest: Any, src: Any, extend_lists: bool = False, remove_duplicates: bool = False) -> None: + from omegaconf import DictConfig, ListConfig, OmegaConf + + assert isinstance(dest, ListConfig) + assert isinstance(src, ListConfig) + + if src._is_none(): + dest._set_value(None) + elif src._is_missing(): + # do not change dest if src is MISSING. + if dest._metadata.element_type is Any: + dest._metadata.element_type = src._metadata.element_type + elif src._is_interpolation(): + dest._set_value(src._value()) + else: + temp_target = ListConfig(content=[], parent=dest._get_parent()) + temp_target.__dict__["_metadata"] = copy.deepcopy(dest.__dict__["_metadata"]) + is_optional, et = _resolve_optional(dest._metadata.element_type) + if is_structured_config(et): + prototype = DictConfig(et, ref_type=et, is_optional=is_optional) + for item in src._iter_ex(resolve=False): + if isinstance(item, DictConfig): + item = OmegaConf.merge(prototype, item) + temp_target.append(item) + else: + for item in src._iter_ex(resolve=False): + temp_target.append(item) + + if extend_lists: + if remove_duplicates: + for entry in temp_target.__dict__["_content"]: + if entry not in dest.__dict__["_content"]: + dest.__dict__["_content"].append(entry) + else: + dest.__dict__["_content"].extend(temp_target.__dict__["_content"]) + else: + dest.__dict__["_content"] = temp_target.__dict__["_content"] + + # explicit flags on the source config are replacing the flag values in the destination + flags = src._metadata.flags + assert flags is not None + for flag, value in flags.items(): + if value is not None: + dest._set_flag(flag, value) + + def merge_with( + self, + *others: Union["BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any], + extend_lists: bool = False, + remove_duplicates: bool = False, + ) -> None: + try: + self._merge_with(*others, extend_lists=extend_lists, remove_duplicates=remove_duplicates) + except Exception as e: + self._format_and_raise(key=None, value=None, cause=e) + + def _merge_with( + self, + *others: Union["BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any], + extend_lists: bool = False, + remove_duplicates: bool = False, + ) -> None: + from .dictconfig import DictConfig + from .listconfig import ListConfig + + """merge a list of other Config objects into this one, overriding as needed""" + for other in others: + if other is None: + raise ValueError("Cannot merge with a None config") + + my_flags = {} + if self._get_flag("allow_objects") is True: + my_flags = {"allow_objects": True} + other = _ensure_container(other, flags=my_flags) + + if isinstance(self, DictConfig) and isinstance(other, DictConfig): + BaseContainer._map_merge( + self, + other, + extend_lists=extend_lists, + remove_duplicates=remove_duplicates, + ) + elif isinstance(self, ListConfig) and isinstance(other, ListConfig): + BaseContainer._list_merge( + self, + other, + extend_lists=extend_lists, + remove_duplicates=remove_duplicates, + ) + else: + raise TypeError("Cannot merge DictConfig with ListConfig") + + # recursively correct the parent hierarchy after the merge + self._re_parent() + + # noinspection PyProtectedMember + def _set_item_impl(self, key: Any, value: Any) -> None: + """ + Changes the value of the node key with the desired value. If the node key doesn't + exist it creates a new one. + """ + from .nodes import AnyNode, ValueNode + + if isinstance(value, Node): + do_deepcopy = not self._get_flag("no_deepcopy_set_nodes") + if not do_deepcopy and isinstance(value, Box): + # if value is from the same config, perform a deepcopy no matter what. + if self._get_root() is value._get_root(): + do_deepcopy = True + + if do_deepcopy: + value = copy.deepcopy(value) + value._set_parent(None) + + try: + old = value._key() + value._set_key(key) + self._validate_set(key, value) + finally: + value._set_key(old) + else: + self._validate_set(key, value) + + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot change read-only config container") + + input_is_node = isinstance(value, Node) + target_node_ref = self._get_node(key) + assert target_node_ref is None or isinstance(target_node_ref, Node) + + input_is_typed_vnode = isinstance(value, ValueNode) and not isinstance(value, AnyNode) + + def get_target_type_hint(val: Any) -> Any: + if not is_structured_config(val): + type_hint = self._metadata.element_type + else: + target = self._get_node(key) + if target is None: + type_hint = self._metadata.element_type + else: + assert isinstance(target, Node) + type_hint = target._metadata.type_hint + return type_hint + + target_type_hint = get_target_type_hint(value) + _, target_ref_type = _resolve_optional(target_type_hint) + + def assign(value_key: Any, val: Node) -> None: + assert val._get_parent() is None + v = val + v._set_parent(self) + v._set_key(value_key) + _deep_update_type_hint(node=v, type_hint=self._metadata.element_type) + self.__dict__["_content"][value_key] = v + + if input_is_typed_vnode and not is_union_annotation(target_ref_type): + assign(key, value) + else: + # input is not a ValueNode, can be primitive or box + + special_value = _is_special(value) + # We use the `Node._set_value` method if the target node exists and: + # 1. the target has an explicit ref_type, or + # 2. the target is an AnyNode and the input is a primitive type. + should_set_value = target_node_ref is not None and ( + target_node_ref._has_ref_type() + or (isinstance(target_node_ref, AnyNode) and is_primitive_type_annotation(value)) + ) + if should_set_value: + if special_value and isinstance(value, Node): + value = value._value() + self.__dict__["_content"][key]._set_value(value) + elif input_is_node: + if ( + special_value + and (is_container_annotation(target_ref_type) or is_structured_config(target_ref_type)) + or is_primitive_type_annotation(target_ref_type) + or is_union_annotation(target_ref_type) + ): + value = _get_value(value) + self._wrap_value_and_set(key, value, target_type_hint) + else: + assign(key, value) + else: + self._wrap_value_and_set(key, value, target_type_hint) + + def _wrap_value_and_set(self, key: Any, val: Any, type_hint: Any) -> None: + from omegaconf.omegaconf import _maybe_wrap + + is_optional, ref_type = _resolve_optional(type_hint) + + try: + wrapped = _maybe_wrap( + ref_type=ref_type, + key=key, + value=val, + is_optional=is_optional, + parent=self, + ) + except ValidationError as e: + self._format_and_raise(key=key, value=val, cause=e) + self.__dict__["_content"][key] = wrapped + + @staticmethod + def _item_eq( + c1: Container, + k1: Union[DictKeyType, int], + c2: Container, + k2: Union[DictKeyType, int], + ) -> bool: + v1 = c1._get_child(k1) + v2 = c2._get_child(k2) + assert v1 is not None and v2 is not None + + assert isinstance(v1, Node) + assert isinstance(v2, Node) + + if v1._is_none() and v2._is_none(): + return True + + if v1._is_missing() and v2._is_missing(): + return True + + v1_inter = v1._is_interpolation() + v2_inter = v2._is_interpolation() + dv1: Optional[Node] = v1 + dv2: Optional[Node] = v2 + + if v1_inter: + dv1 = v1._maybe_dereference_node() + if v2_inter: + dv2 = v2._maybe_dereference_node() + + if v1_inter and v2_inter: + if dv1 is None or dv2 is None: + return v1 == v2 + else: + # both are not none, if both are containers compare as container + if isinstance(dv1, Container) and isinstance(dv2, Container): + if dv1 != dv2: + return False + dv1 = _get_value(dv1) + dv2 = _get_value(dv2) + return dv1 == dv2 + elif not v1_inter and not v2_inter: + v1 = _get_value(v1) + v2 = _get_value(v2) + ret = v1 == v2 + assert isinstance(ret, bool) + return ret + else: + dv1 = _get_value(dv1) + dv2 = _get_value(dv2) + ret = dv1 == dv2 + assert isinstance(ret, bool) + return ret + + def _is_optional(self) -> bool: + return self.__dict__["_metadata"].optional is True + + def _is_interpolation(self) -> bool: + return _is_interpolation(self.__dict__["_content"]) + + @abstractmethod + def _validate_get(self, key: Any, value: Any = None) -> None: + ... + + @abstractmethod + def _validate_set(self, key: Any, value: Any) -> None: + ... + + def _value(self) -> Any: + return self.__dict__["_content"] + + def _get_full_key(self, key: Union[DictKeyType, int, slice, None]) -> str: + from .listconfig import ListConfig + from .omegaconf import _select_one + + if not isinstance(key, (int, str, Enum, float, bool, slice, bytes, type(None))): + return "" + + def _slice_to_str(x: slice) -> str: + if x.step is not None: + return f"{x.start}:{x.stop}:{x.step}" + else: + return f"{x.start}:{x.stop}" + + def prepand( + full_key: str, + parent_type: Any, + cur_type: Any, + key: Optional[Union[DictKeyType, int, slice]], + ) -> str: + if key is None: + return full_key + + if isinstance(key, slice): + key = _slice_to_str(key) + elif isinstance(key, Enum): + key = key.name + else: + key = str(key) + + assert isinstance(key, str) + + if issubclass(parent_type, ListConfig): + if full_key != "": + if issubclass(cur_type, ListConfig): + full_key = f"[{key}]{full_key}" + else: + full_key = f"[{key}].{full_key}" + else: + full_key = f"[{key}]" + else: + if full_key == "": + full_key = key + else: + if issubclass(cur_type, ListConfig): + full_key = f"{key}{full_key}" + else: + full_key = f"{key}.{full_key}" + return full_key + + if key is not None and key != "": + assert isinstance(self, Container) + cur, _ = _select_one(c=self, key=str(key), throw_on_missing=False, throw_on_type_error=False) + if cur is None: + cur = self + full_key = prepand("", type(cur), None, key) + if cur._key() is not None: + full_key = prepand(full_key, type(cur._get_parent()), type(cur), cur._key()) + else: + full_key = prepand("", type(cur._get_parent()), type(cur), cur._key()) + else: + cur = self + if cur._key() is None: + return "" + full_key = self._key() + + assert cur is not None + memo = {id(cur)} # remember already visited nodes so as to detect cycles + while cur._get_parent() is not None: + cur = cur._get_parent() + if id(cur) in memo: + raise ConfigCycleDetectedException(f"Cycle when iterating over parents of key `{key!s}`") + memo.add(id(cur)) + assert cur is not None + if cur._key() is not None: + full_key = prepand(full_key, type(cur._get_parent()), type(cur), cur._key()) + + return full_key + + +def _create_structured_with_missing_fields( + ref_type: type, object_type: Optional[type] = None +) -> "DictConfig": + from . import MISSING, DictConfig + + cfg_data = get_structured_config_data(ref_type) + for v in cfg_data.values(): + v._set_value(MISSING) + + cfg = DictConfig(cfg_data) + cfg._metadata.optional, cfg._metadata.ref_type = _resolve_optional(ref_type) + cfg._metadata.object_type = object_type + + return cfg + + +def _update_types(node: Node, ref_type: Any, object_type: Optional[type]) -> None: + if object_type is not None and not is_primitive_dict(object_type): + node._metadata.object_type = object_type + + if node._metadata.ref_type is Any: + _deep_update_type_hint(node, ref_type) + + +def _deep_update_type_hint(node: Node, type_hint: Any) -> None: + """Ensure node is compatible with type_hint, mutating if necessary.""" + from omegaconf import DictConfig, ListConfig + + from ._utils import get_dict_key_value_types, get_list_element_type + + if type_hint is Any: + return + + _shallow_validate_type_hint(node, type_hint) + + new_is_optional, new_ref_type = _resolve_optional(type_hint) + node._metadata.ref_type = new_ref_type + node._metadata.optional = new_is_optional + + if is_list_annotation(new_ref_type) and isinstance(node, ListConfig): + new_element_type = get_list_element_type(new_ref_type) + node._metadata.element_type = new_element_type + if not _is_special(node): + for i in range(len(node)): + _deep_update_subnode(node, i, new_element_type) + + if is_dict_annotation(new_ref_type) and isinstance(node, DictConfig): + new_key_type, new_element_type = get_dict_key_value_types(new_ref_type) + node._metadata.key_type = new_key_type + node._metadata.element_type = new_element_type + if not _is_special(node): + for key in node: + if new_key_type is not Any and not isinstance(key, new_key_type): + raise KeyValidationError( + f"Key {key!r} ({type(key).__name__}) is incompatible" + + f" with key type hint '{new_key_type.__name__}'" + ) + _deep_update_subnode(node, key, new_element_type) + + +def _deep_update_subnode(node: BaseContainer, key: Any, value_type_hint: Any) -> None: + """Get node[key] and ensure it is compatible with value_type_hint, mutating if necessary.""" + subnode = node._get_node(key) + assert isinstance(subnode, Node) + if _is_special(subnode): + # Ensure special values are wrapped in a Node subclass that + # is compatible with the type hint. + node._wrap_value_and_set(key, subnode._value(), value_type_hint) + subnode = node._get_node(key) + assert isinstance(subnode, Node) + _deep_update_type_hint(subnode, value_type_hint) + + +def _shallow_validate_type_hint(node: Node, type_hint: Any) -> None: + """Error if node's type, content and metadata are not compatible with type_hint.""" + from omegaconf import DictConfig, ListConfig, ValueNode + + is_optional, ref_type = _resolve_optional(type_hint) + + vk = get_value_kind(node) + + if node._is_none(): + if not is_optional: + value = _get_value(node) + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type.__name__}'" + ) + return + elif vk in (ValueKind.MANDATORY_MISSING, ValueKind.INTERPOLATION): + return + elif vk == ValueKind.VALUE: + if is_primitive_type_annotation(ref_type) and isinstance(node, ValueNode): + value = node._value() + if not isinstance(value, ref_type): + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type.__name__}'" + ) + elif is_structured_config(ref_type) and isinstance(node, DictConfig): + return + elif is_dict_annotation(ref_type) and isinstance(node, DictConfig): + return + elif is_list_annotation(ref_type) and isinstance(node, ListConfig): + return + else: + if isinstance(node, ValueNode): + value = node._value() + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type}'" + ) + else: + raise ValidationError( + f"'{type(node).__name__}' is incompatible" + f" with type hint '{ref_type}'" + ) + + else: + assert False diff --git a/omegaconf/dictconfig.py b/omegaconf/dictconfig.py new file mode 100644 index 000000000..9d8310a99 --- /dev/null +++ b/omegaconf/dictconfig.py @@ -0,0 +1,727 @@ +import copy +from enum import Enum +from typing import ( + Any, + Dict, + ItemsView, + Iterable, + Iterator, + KeysView, + List, + MutableMapping, + Optional, + Sequence, + Tuple, + Type, + Union, +) + +from ._utils import ( + _DEFAULT_MARKER_, + ValueKind, + _get_value, + _is_interpolation, + _is_missing_literal, + _is_missing_value, + _is_none, + _resolve_optional, + _valid_dict_key_annotation_type, + format_and_raise, + get_structured_config_data, + get_structured_config_init_field_names, + get_type_of, + get_value_kind, + is_container_annotation, + is_dict, + is_primitive_dict, + is_structured_config, + is_structured_config_frozen, + type_str, +) +from .base import Box, Container, ContainerMetadata, DictKeyType, Node +from .basecontainer import BaseContainer +from .errors import ( + ConfigAttributeError, + ConfigKeyError, + ConfigTypeError, + InterpolationResolutionError, + KeyValidationError, + MissingMandatoryValue, + OmegaConfBaseException, + ReadonlyConfigError, + ValidationError, +) +from .nodes import EnumNode, ValueNode + + +class DictConfig(BaseContainer, MutableMapping[Any, Any]): + _metadata: ContainerMetadata + _content: Union[Dict[DictKeyType, Node], None, str] + + def __init__( + self, + content: Union[Dict[DictKeyType, Any], "DictConfig", Any], + key: Any = None, + parent: Optional[Box] = None, + ref_type: Union[Any, Type[Any]] = Any, + key_type: Union[Any, Type[Any]] = Any, + element_type: Union[Any, Type[Any]] = Any, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ) -> None: + try: + if isinstance(content, DictConfig): + if flags is None: + flags = content._metadata.flags + super().__init__( + parent=parent, + metadata=ContainerMetadata( + key=key, + optional=is_optional, + ref_type=ref_type, + object_type=dict, + key_type=key_type, + element_type=element_type, + flags=flags, + ), + ) + + if not _valid_dict_key_annotation_type(key_type): + raise KeyValidationError(f"Unsupported key type {key_type}") + + if is_structured_config(content) or is_structured_config(ref_type): + self._set_value(content, flags=flags) + if is_structured_config_frozen(content) or is_structured_config_frozen(ref_type): + self._set_flag("readonly", True) + + else: + if isinstance(content, DictConfig): + metadata = copy.deepcopy(content._metadata) + metadata.key = key + metadata.ref_type = ref_type + metadata.optional = is_optional + metadata.element_type = element_type + metadata.key_type = key_type + self.__dict__["_metadata"] = metadata + self._set_value(content, flags=flags) + except Exception as ex: + format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex)) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "DictConfig": + res = DictConfig(None) + res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo) + res.__dict__["_flags_cache"] = copy.deepcopy(self.__dict__["_flags_cache"], memo=memo) + + src_content = self.__dict__["_content"] + if isinstance(src_content, dict): + content_copy = {} + for k, v in src_content.items(): + old_parent = v.__dict__["_parent"] + try: + v.__dict__["_parent"] = None + vc = copy.deepcopy(v, memo=memo) + vc.__dict__["_parent"] = res + content_copy[k] = vc + finally: + v.__dict__["_parent"] = old_parent + else: + # None and strings can be assigned as is + content_copy = src_content + + res.__dict__["_content"] = content_copy + # parent is retained, but not copied + res.__dict__["_parent"] = self.__dict__["_parent"] + return res + + def copy(self) -> "DictConfig": + return copy.copy(self) + + def _is_typed(self) -> bool: + return self._metadata.object_type not in (Any, None) and not is_dict(self._metadata.object_type) + + def _validate_get(self, key: Any, value: Any = None) -> None: + is_typed = self._is_typed() + + is_struct = self._get_flag("struct") is True + if key not in self.__dict__["_content"]: + if is_typed: + # do not raise an exception if struct is explicitly set to False + if self._get_node_flag("struct") is False: + return + if is_typed or is_struct: + if is_typed: + assert self._metadata.object_type not in (dict, None) + msg = f"Key '{key}' not in '{self._metadata.object_type.__name__}'" + else: + msg = f"Key '{key}' is not in struct" + self._format_and_raise(key=key, value=value, cause=ConfigAttributeError(msg)) + + def _validate_set(self, key: Any, value: Any) -> None: + from omegaconf import OmegaConf + + vk = get_value_kind(value) + if vk == ValueKind.INTERPOLATION: + return + if _is_none(value): + self._validate_non_optional(key, value) + return + if vk == ValueKind.MANDATORY_MISSING or value is None: + return + + target = self._get_node(key) if key is not None else self + + target_has_ref_type = isinstance(target, DictConfig) and target._metadata.ref_type not in (Any, dict) + is_valid_target = target is None or not target_has_ref_type + + if is_valid_target: + return + + assert isinstance(target, Node) + + target_type = target._metadata.ref_type + value_type = OmegaConf.get_type(value) + + if is_dict(value_type) and is_dict(target_type): + return + if is_container_annotation(target_type) and not is_container_annotation(value_type): + raise ValidationError(f"Cannot assign {type_str(value_type)} to {type_str(target_type)}") + + if target_type is not None and value_type is not None: + origin = getattr(target_type, "__origin__", target_type) + if not issubclass(value_type, origin): + self._raise_invalid_value(value, value_type, target_type) + + def _validate_merge(self, value: Any) -> None: + from omegaconf import OmegaConf + + dest = self + src = value + + self._validate_non_optional(None, src) + + dest_obj_type = OmegaConf.get_type(dest) + src_obj_type = OmegaConf.get_type(src) + + if dest._is_missing() and src._metadata.object_type not in (dict, None): + self._validate_set(key=None, value=_get_value(src)) + + if src._is_missing(): + return + + validation_error = ( + dest_obj_type is not None + and src_obj_type is not None + and is_structured_config(dest_obj_type) + and not src._is_none() + and not is_dict(src_obj_type) + and not issubclass(src_obj_type, dest_obj_type) + ) + if validation_error: + msg = ( + f"Merge error: {type_str(src_obj_type)} is not a " + f"subclass of {type_str(dest_obj_type)}. value: {src}" + ) + raise ValidationError(msg) + + def _validate_non_optional(self, key: Optional[DictKeyType], value: Any) -> None: + if _is_none(value, resolve=True, throw_on_resolution_failure=False): + if key is not None: + child = self._get_node(key) + if child is not None: + assert isinstance(child, Node) + field_is_optional = child._is_optional() + else: + field_is_optional, _ = _resolve_optional(self._metadata.element_type) + else: + field_is_optional = self._is_optional() + + if not field_is_optional: + self._format_and_raise( + key=key, + value=value, + cause=ValidationError("field '$FULL_KEY' is not Optional"), + ) + + def _raise_invalid_value(self, value: Any, value_type: Any, target_type: Any) -> None: + assert value_type is not None + assert target_type is not None + msg = ( + f"Invalid type assigned: {type_str(value_type)} is not a " + f"subclass of {type_str(target_type)}. value: {value}" + ) + raise ValidationError(msg) + + def _validate_and_normalize_key(self, key: Any) -> DictKeyType: + return self._s_validate_and_normalize_key(self._metadata.key_type, key) + + def _s_validate_and_normalize_key(self, key_type: Any, key: Any) -> DictKeyType: + if key_type is Any: + for t in DictKeyType.__args__: # type: ignore + if isinstance(key, t): + return key # type: ignore + raise KeyValidationError("Incompatible key type '$KEY_TYPE'") + elif key_type is bool and key in [0, 1]: + # Python treats True as 1 and False as 0 when used as dict keys + # assert hash(0) == hash(False) + # assert hash(1) == hash(True) + return bool(key) + elif key_type in (str, bytes, int, float, bool): # primitive type + if not isinstance(key, key_type): + raise KeyValidationError(f"Key $KEY ($KEY_TYPE) is incompatible with ({key_type.__name__})") + + return key # type: ignore + elif issubclass(key_type, Enum): + try: + return EnumNode.validate_and_convert_to_enum(key_type, key) + except ValidationError: + valid = ", ".join([x for x in key_type.__members__.keys()]) + raise KeyValidationError( + f"Key '$KEY' is incompatible with the enum type '{key_type.__name__}', valid: [{valid}]" + ) + else: + assert False, f"Unsupported key type {key_type}" + + def __setitem__(self, key: DictKeyType, value: Any) -> None: + try: + self.__set_impl(key=key, value=value) + except AttributeError as e: + self._format_and_raise(key=key, value=value, type_override=ConfigKeyError, cause=e) + except Exception as e: + self._format_and_raise(key=key, value=value, cause=e) + + def __set_impl(self, key: DictKeyType, value: Any) -> None: + key = self._validate_and_normalize_key(key) + self._set_item_impl(key, value) + + # hide content while inspecting in debugger + def __dir__(self) -> Iterable[str]: + if self._is_missing() or self._is_none(): + return [] + return self.__dict__["_content"].keys() # type: ignore + + def __setattr__(self, key: str, value: Any) -> None: + """ + Allow assigning attributes to DictConfig + :param key: + :param value: + :return: + """ + try: + self.__set_impl(key, value) + except Exception as e: + if isinstance(e, OmegaConfBaseException) and e._initialized: + raise e + self._format_and_raise(key=key, value=value, cause=e) + assert False + + def __getattr__(self, key: str) -> Any: + """ + Allow accessing dictionary values as attributes + :param key: + :return: + """ + if key == "__name__": + raise AttributeError() + + try: + return self._get_impl(key=key, default_value=_DEFAULT_MARKER_, validate_key=False) + except ConfigKeyError as e: + self._format_and_raise(key=key, value=None, cause=e, type_override=ConfigAttributeError) + except Exception as e: + self._format_and_raise(key=key, value=None, cause=e) + + def __getitem__(self, key: DictKeyType) -> Any: + """ + Allow map style access + :param key: + :return: + """ + + try: + return self._get_impl(key=key, default_value=_DEFAULT_MARKER_) + except AttributeError as e: + self._format_and_raise(key=key, value=None, cause=e, type_override=ConfigKeyError) + except Exception as e: + self._format_and_raise(key=key, value=None, cause=e) + + def __delattr__(self, key: str) -> None: + """ + Allow deleting dictionary values as attributes + :param key: + :return: + """ + if self._get_flag("readonly"): + self._format_and_raise( + key=key, + value=None, + cause=ReadonlyConfigError("DictConfig in read-only mode does not support deletion"), + ) + try: + del self.__dict__["_content"][key] + except KeyError: + msg = "Attribute not found: '$KEY'" + self._format_and_raise(key=key, value=None, cause=ConfigAttributeError(msg)) + + def __delitem__(self, key: DictKeyType) -> None: + key = self._validate_and_normalize_key(key) + if self._get_flag("readonly"): + self._format_and_raise( + key=key, + value=None, + cause=ReadonlyConfigError("DictConfig in read-only mode does not support deletion"), + ) + if self._get_flag("struct"): + self._format_and_raise( + key=key, + value=None, + cause=ConfigTypeError("DictConfig in struct mode does not support deletion"), + ) + if self._is_typed() and self._get_node_flag("struct") is not False: + self._format_and_raise( + key=key, + value=None, + cause=ConfigTypeError( + f"{type_str(self._metadata.object_type)} (DictConfig) does not support deletion" + ), + ) + + try: + del self.__dict__["_content"][key] + except KeyError: + msg = "Key not found: '$KEY'" + self._format_and_raise(key=key, value=None, cause=ConfigKeyError(msg)) + + def get(self, key: DictKeyType, default_value: Any = None) -> Any: + """Return the value for `key` if `key` is in the dictionary, else + `default_value` (defaulting to `None`).""" + try: + return self._get_impl(key=key, default_value=default_value) + except KeyValidationError as e: + self._format_and_raise(key=key, value=None, cause=e) + + def _get_impl(self, key: DictKeyType, default_value: Any, validate_key: bool = True) -> Any: + try: + node = self._get_child(key=key, throw_on_missing_key=True, validate_key=validate_key) + except (ConfigAttributeError, ConfigKeyError): + if default_value is not _DEFAULT_MARKER_: + return default_value + else: + raise + assert isinstance(node, Node) + return self._resolve_with_default(key=key, value=node, default_value=default_value) + + def _get_node( + self, + key: DictKeyType, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Optional[Node]: + try: + key = self._validate_and_normalize_key(key) + except KeyValidationError: + if validate_access and validate_key: + raise + else: + if throw_on_missing_key: + raise ConfigAttributeError + else: + return None + + if validate_access: + self._validate_get(key) + + value: Optional[Node] = self.__dict__["_content"].get(key) + if value is None: + if throw_on_missing_key: + raise ConfigKeyError(f"Missing key {key!s}") + elif throw_on_missing_value and value._is_missing(): + raise MissingMandatoryValue("Missing mandatory value: $KEY") + return value + + def pop(self, key: DictKeyType, default: Any = _DEFAULT_MARKER_) -> Any: + try: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot pop from read-only node") + if self._get_flag("struct"): + raise ConfigTypeError("DictConfig in struct mode does not support pop") + if self._is_typed() and self._get_node_flag("struct") is not False: + raise ConfigTypeError( + f"{type_str(self._metadata.object_type)} (DictConfig) does not support pop" + ) + key = self._validate_and_normalize_key(key) + node = self._get_child(key=key, validate_access=False) + if node is not None: + assert isinstance(node, Node) + value = self._resolve_with_default(key=key, value=node, default_value=default) + + del self[key] + return value + else: + if default is not _DEFAULT_MARKER_: + return default + else: + full = self._get_full_key(key=key) + if full != key: + raise ConfigKeyError(f"Key not found: '{key!s}' (path: '{full}')") + else: + raise ConfigKeyError(f"Key not found: '{key!s}'") + except Exception as e: + self._format_and_raise(key=key, value=None, cause=e) + + def keys(self) -> KeysView[DictKeyType]: + if self._is_missing() or self._is_interpolation() or self._is_none(): + return {}.keys() + ret = self.__dict__["_content"].keys() + assert isinstance(ret, KeysView) + return ret + + def __contains__(self, key: object) -> bool: + """ + A key is contained in a DictConfig if there is an associated value and + it is not a mandatory missing value ('???'). + :param key: + :return: + """ + + try: + key = self._validate_and_normalize_key(key) + except KeyValidationError: + return False + + try: + node = self._get_child(key) + assert node is None or isinstance(node, Node) + except (KeyError, AttributeError): + node = None + + if node is None: + return False + else: + try: + self._resolve_with_default(key=key, value=node) + return True + except InterpolationResolutionError: + # Interpolations that fail count as existing. + return True + except MissingMandatoryValue: + # Missing values count as *not* existing. + return False + + def __iter__(self) -> Iterator[DictKeyType]: + return iter(self.keys()) + + def items(self) -> ItemsView[DictKeyType, Any]: + return dict(self.items_ex(resolve=True, keys=None)).items() + + def setdefault(self, key: DictKeyType, default: Any = None) -> Any: + if key in self: + ret = self.__getitem__(key) + else: + ret = default + self.__setitem__(key, default) + return ret + + def items_ex( + self, resolve: bool = True, keys: Optional[Sequence[DictKeyType]] = None + ) -> List[Tuple[DictKeyType, Any]]: + items: List[Tuple[DictKeyType, Any]] = [] + + if self._is_none(): + self._format_and_raise( + key=None, + value=None, + cause=TypeError("Cannot iterate a DictConfig object representing None"), + ) + if self._is_missing(): + raise MissingMandatoryValue("Cannot iterate a missing DictConfig") + + for key in self.keys(): + if resolve: + value = self[key] + else: + value = self.__dict__["_content"][key] + if isinstance(value, ValueNode): + value = value._value() + if keys is None or key in keys: + items.append((key, value)) + + return items + + def __eq__(self, other: Any) -> bool: + if other is None: + return self.__dict__["_content"] is None + if is_primitive_dict(other) or is_structured_config(other): + other = DictConfig(other, flags={"allow_objects": True}) + return DictConfig._dict_conf_eq(self, other) + if isinstance(other, DictConfig): + return DictConfig._dict_conf_eq(self, other) + if self._is_missing(): + return _is_missing_literal(other) + return NotImplemented + + def __ne__(self, other: Any) -> bool: + x = self.__eq__(other) + if x is not NotImplemented: + return not x + return NotImplemented + + def __hash__(self) -> int: + return hash(str(self)) + + def _promote(self, type_or_prototype: Optional[Type[Any]]) -> None: + """ + Retypes a node. + This should only be used in rare circumstances, where you want to dynamically change + the runtime structured-type of a DictConfig. + It will change the type and add the additional fields based on the input class or object + """ + if type_or_prototype is None: + return + if not is_structured_config(type_or_prototype): + raise ValueError(f"Expected structured config class: {type_or_prototype}") + + from omegaconf import OmegaConf + + proto: DictConfig = OmegaConf.structured(type_or_prototype) + object_type = proto._metadata.object_type + # remove the type to prevent assignment validation from rejecting the promotion. + proto._metadata.object_type = None + self.merge_with(proto) + # restore the type. + self._metadata.object_type = object_type + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + try: + previous_content = self.__dict__["_content"] + self._set_value_impl(value, flags) + except Exception as e: + self.__dict__["_content"] = previous_content + raise e + + def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + from omegaconf import MISSING, flag_override + + if flags is None: + flags = {} + + assert not isinstance(value, ValueNode) + self._validate_set(key=None, value=value) + + if _is_none(value, resolve=True): + self.__dict__["_content"] = None + self._metadata.object_type = None + elif _is_interpolation(value, strict_interpolation_validation=True): + self.__dict__["_content"] = value + self._metadata.object_type = None + elif _is_missing_value(value): + self.__dict__["_content"] = MISSING + self._metadata.object_type = None + else: + self.__dict__["_content"] = {} + if is_structured_config(value): + self._metadata.object_type = None + ao = self._get_flag("allow_objects") + data = get_structured_config_data(value, allow_objects=ao) + with flag_override(self, ["struct", "readonly"], False): + for k, v in data.items(): + self.__setitem__(k, v) + self._metadata.object_type = get_type_of(value) + + elif isinstance(value, DictConfig): + self._metadata.flags = copy.deepcopy(flags) + with flag_override(self, ["struct", "readonly"], False): + for k, v in value.__dict__["_content"].items(): + self.__setitem__(k, v) + self._metadata.object_type = value._metadata.object_type + + elif isinstance(value, dict): + with flag_override(self, ["struct", "readonly"], False): + for k, v in value.items(): + self.__setitem__(k, v) + self._metadata.object_type = dict + + else: # pragma: no cover + msg = f"Unsupported value type: {value}" + raise ValidationError(msg) + + @staticmethod + def _dict_conf_eq(d1: "DictConfig", d2: "DictConfig") -> bool: + d1_none = d1.__dict__["_content"] is None + d2_none = d2.__dict__["_content"] is None + if d1_none and d2_none: + return True + if d1_none != d2_none: + return False + + assert isinstance(d1, DictConfig) + assert isinstance(d2, DictConfig) + if len(d1) != len(d2): + return False + if d1._is_missing() or d2._is_missing(): + return d1._is_missing() is d2._is_missing() + + for k, v in d1.items_ex(resolve=False): + if k not in d2.__dict__["_content"]: + return False + if not BaseContainer._item_eq(d1, k, d2, k): + return False + + return True + + def _to_object(self) -> Any: + """ + Instantiate an instance of `self._metadata.object_type`. + This requires `self` to be a structured config. + Nested subconfigs are converted by calling `OmegaConf.to_object`. + """ + from omegaconf import OmegaConf + + object_type = self._metadata.object_type + assert is_structured_config(object_type) + init_field_names = set(get_structured_config_init_field_names(object_type)) + + init_field_items: Dict[str, Any] = {} + non_init_field_items: Dict[str, Any] = {} + for k in self.keys(): + assert isinstance(k, str) + node = self._get_child(k) + assert isinstance(node, Node) + try: + node = node._dereference_node() + except InterpolationResolutionError as e: + self._format_and_raise(key=k, value=None, cause=e) + if node._is_missing(): + if k not in init_field_names: + continue # MISSING is ignored for init=False fields + self._format_and_raise( + key=k, + value=None, + cause=MissingMandatoryValue( + "Structured config of type `$OBJECT_TYPE` has missing mandatory value: $KEY" + ), + ) + if isinstance(node, Container): + v = OmegaConf.to_object(node) + else: + v = node._value() + + if k in init_field_names: + init_field_items[k] = v + else: + non_init_field_items[k] = v + + try: + result = object_type(**init_field_items) + except TypeError as exc: + self._format_and_raise( + key=None, + value=None, + cause=exc, + msg="Could not create instance of `$OBJECT_TYPE`: " + str(exc), + ) + + for k, v in non_init_field_items.items(): + setattr(result, k, v) + return result diff --git a/omegaconf/errors.py b/omegaconf/errors.py new file mode 100644 index 000000000..60525ef3c --- /dev/null +++ b/omegaconf/errors.py @@ -0,0 +1,141 @@ +from typing import Any, Optional, Type + + +class OmegaConfBaseException(Exception): + # would ideally be typed Optional[Node] + parent_node: Any + child_node: Any + key: Any + full_key: Optional[str] + value: Any + msg: Optional[str] + cause: Optional[Exception] + object_type: Optional[Type[Any]] + object_type_str: Optional[str] + ref_type: Optional[Type[Any]] + ref_type_str: Optional[str] + + _initialized: bool = False + + def __init__(self, *_args: Any, **_kwargs: Any) -> None: + self.parent_node = None + self.child_node = None + self.key = None + self.full_key = None + self.value = None + self.msg = None + self.object_type = None + self.ref_type = None + + +class MissingMandatoryValue(OmegaConfBaseException): + """Thrown when a variable flagged with '???' value is accessed to + indicate that the value was not set""" + + +class KeyValidationError(OmegaConfBaseException, ValueError): + """ + Thrown when an a key of invalid type is used + """ + + +class ValidationError(OmegaConfBaseException, ValueError): + """ + Thrown when a value fails validation + """ + + +class UnsupportedValueType(ValidationError, ValueError): + """ + Thrown when an input value is not of supported type + """ + + +class ReadonlyConfigError(OmegaConfBaseException): + """ + Thrown when someone tries to modify a frozen config + """ + + +class InterpolationResolutionError(OmegaConfBaseException, ValueError): + """ + Base class for exceptions raised when resolving an interpolation. + """ + + +class UnsupportedInterpolationType(InterpolationResolutionError): + """ + Thrown when an attempt to use an unregistered interpolation is made + """ + + +class InterpolationKeyError(InterpolationResolutionError): + """ + Thrown when a node does not exist when resolving an interpolation. + """ + + +class InterpolationToMissingValueError(InterpolationResolutionError): + """ + Thrown when a node interpolation points to a node that is set to ???. + """ + + +class InterpolationValidationError(InterpolationResolutionError, ValidationError): + """ + Thrown when the result of an interpolation fails the validation step. + """ + + +class ConfigKeyError(OmegaConfBaseException, KeyError): + """ + Thrown from DictConfig when a regular dict access would have caused a KeyError. + """ + + msg: str + + def __init__(self, msg: str) -> None: + super().__init__(msg) + self.msg = msg + + def __str__(self) -> str: + """ + Workaround to nasty KeyError quirk: https://bugs.python.org/issue2651 + """ + return self.msg + + +class ConfigAttributeError(OmegaConfBaseException, AttributeError): + """ + Thrown from a config object when a regular access would have caused an AttributeError. + """ + + +class ConfigTypeError(OmegaConfBaseException, TypeError): + """ + Thrown from a config object when a regular access would have caused a TypeError. + """ + + +class ConfigIndexError(OmegaConfBaseException, IndexError): + """ + Thrown from a config object when a regular access would have caused an IndexError. + """ + + +class ConfigValueError(OmegaConfBaseException, ValueError): + """ + Thrown from a config object when a regular access would have caused a ValueError. + """ + + +class ConfigCycleDetectedException(OmegaConfBaseException): + """ + Thrown when a cycle is detected in the graph made by config nodes. + """ + + +class GrammarParseError(OmegaConfBaseException): + """ + Thrown when failing to parse an expression according to the ANTLR grammar. + """ diff --git a/omegaconf/grammar/OmegaConfGrammarLexer.g4 b/omegaconf/grammar/OmegaConfGrammarLexer.g4 new file mode 100644 index 000000000..f7ab2044a --- /dev/null +++ b/omegaconf/grammar/OmegaConfGrammarLexer.g4 @@ -0,0 +1,137 @@ +// Regenerate lexer and parser by running 'python setup.py antlr' at project root. +// See `OmegaConfGrammarParser.g4` for some important information regarding how to +// properly maintain this grammar. + +lexer grammar OmegaConfGrammarLexer; + +// Re-usable fragments. +fragment CHAR: [a-zA-Z]; +fragment DIGIT: [0-9]; +fragment INT_UNSIGNED: '0' | [1-9] (('_')? DIGIT)*; +fragment ESC_BACKSLASH: '\\\\'; // escaped backslash + +///////////////////////////// +// DEFAULT_MODE (TOPLEVEL) // +///////////////////////////// + +TOP_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); + +// Regular string: anything that does not contain any $ and does not end with \ +// (this ensures this rule will not consume characters required to recognize other tokens). +ANY_STR: ~[$]* ~[\\$]; + +// Escaped interpolation: '\${', optionally preceded by an even number of \ +ESC_INTER: ESC_BACKSLASH* '\\${'; + +// Backslashes that *may* be escaped (even number). +TOP_ESC: ESC_BACKSLASH+; + +// Other backslashes that will not need escaping (odd number due to not matching the previous rule). +BACKSLASHES: '\\'+ -> type(ANY_STR); + +// The dollar sign must be singled out so that we can recognize interpolations. +DOLLAR: '$' -> type(ANY_STR); + + +//////////////// +// VALUE_MODE // +//////////////// + +mode VALUE_MODE; + +INTER_OPEN: '${' WS? -> pushMode(INTERPOLATION_MODE); +BRACE_OPEN: '{' WS? -> pushMode(VALUE_MODE); // must keep track of braces to detect end of interpolation +BRACE_CLOSE: WS? '}' -> popMode; +QUOTE_OPEN_SINGLE: '\'' -> pushMode(QUOTED_SINGLE_MODE); +QUOTE_OPEN_DOUBLE: '"' -> pushMode(QUOTED_DOUBLE_MODE); + +COMMA: WS? ',' WS?; +BRACKET_OPEN: '[' WS?; +BRACKET_CLOSE: WS? ']'; +COLON: WS? ':' WS?; + +// Numbers. + +fragment POINT_FLOAT: INT_UNSIGNED '.' | INT_UNSIGNED? '.' DIGIT (('_')? DIGIT)*; +fragment EXPONENT_FLOAT: (INT_UNSIGNED | POINT_FLOAT) [eE] [+-]? DIGIT (('_')? DIGIT)*; +FLOAT: [+-]? (POINT_FLOAT | EXPONENT_FLOAT | [Ii][Nn][Ff] | [Nn][Aa][Nn]); +INT: [+-]? INT_UNSIGNED; + +// Other reserved keywords. + +BOOL: + [Tt][Rr][Uu][Ee] // TRUE + | [Ff][Aa][Ll][Ss][Ee]; // FALSE + +NULL: [Nn][Uu][Ll][Ll]; + +UNQUOTED_CHAR: [/\-\\+.$%*@?|]; // other characters allowed in unquoted strings +ID: (CHAR|'_') (CHAR|DIGIT|'_'|'-')*; +ESC: (ESC_BACKSLASH | '\\(' | '\\)' | '\\[' | '\\]' | '\\{' | '\\}' | + '\\:' | '\\=' | '\\,' | '\\ ' | '\\\t')+; +WS: [ \t]+; + + +//////////////////////// +// INTERPOLATION_MODE // +//////////////////////// + +mode INTERPOLATION_MODE; + +NESTED_INTER_OPEN: INTER_OPEN WS? -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); +INTER_COLON: WS? ':' WS? -> type(COLON), mode(VALUE_MODE); +INTER_CLOSE: WS? '}' -> popMode; + +DOT: '.'; +INTER_BRACKET_OPEN: '[' -> type(BRACKET_OPEN); +INTER_BRACKET_CLOSE: ']' -> type(BRACKET_CLOSE); +INTER_ID: ID -> type(ID); + +// Interpolation key, may contain any non special character. +// Note that we can allow '$' because the parser does not support interpolations that +// are only part of a key name, i.e., "${foo${bar}}" is not allowed. As a result, it +// is ok to "consume" all '$' characters within the `INTER_KEY` token. +INTER_KEY: ~[\\{}()[\]:. \t'"]+; + + +//////////////////////// +// QUOTED_SINGLE_MODE // +//////////////////////// + +mode QUOTED_SINGLE_MODE; + +// This mode is very similar to `DEFAULT_MODE` except for the handling of quotes. + +QSINGLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); +MATCHING_QUOTE_CLOSE: '\'' -> popMode; + +// Regular string: anything that does not contain any $ *or quote* and does not end with \ +QSINGLE_STR: ~['$]* ~['\\$] -> type(ANY_STR); + +QSINGLE_ESC_INTER: ESC_INTER -> type(ESC_INTER); + +// Escaped quote (optionally preceded by an even number of backslashes). +QSINGLE_ESC_QUOTE: ESC_BACKSLASH* '\\\'' -> type(ESC); + +QUOTED_ESC: ESC_BACKSLASH+; +QSINGLE_BACKSLASHES: '\\'+ -> type(ANY_STR); +QSINGLE_DOLLAR: '$' -> type(ANY_STR); + + +//////////////////////// +// QUOTED_DOUBLE_MODE // +//////////////////////// + +mode QUOTED_DOUBLE_MODE; + +// Same as `QUOTED_SINGLE_MODE` but for double quotes. + +QDOUBLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); +QDOUBLE_CLOSE: '"' -> type(MATCHING_QUOTE_CLOSE), popMode; + +QDOUBLE_STR: ~["$]* ~["\\$] -> type(ANY_STR); +QDOUBLE_ESC_INTER: ESC_INTER -> type(ESC_INTER); +QDOUBLE_ESC_QUOTE: ESC_BACKSLASH* '\\"' -> type(ESC); +QDOUBLE_ESC: ESC_BACKSLASH+ -> type(QUOTED_ESC); +QDOUBLE_BACKSLASHES: '\\'+ -> type(ANY_STR); +QDOUBLE_DOLLAR: '$' -> type(ANY_STR); diff --git a/omegaconf/grammar/OmegaConfGrammarParser.g4 b/omegaconf/grammar/OmegaConfGrammarParser.g4 new file mode 100644 index 000000000..040692c8d --- /dev/null +++ b/omegaconf/grammar/OmegaConfGrammarParser.g4 @@ -0,0 +1,91 @@ +// Regenerate parser by running 'python setup.py antlr' at project root. + +// Maintenance guidelines when modifying this grammar: +// +// - Consider whether the regex pattern `SIMPLE_INTERPOLATION_PATTERN` found in +// `grammar_parser.py` should be updated as well. +// +// - Update Hydra's grammar accordingly. +// +// - Keep up-to-date the comments in the visitor (in `grammar_visitor.py`) +// that contain grammar excerpts (within each `visit...()` method). +// +// - Remember to update the documentation (including the tutorial notebook as +// well as grammar.rst) + +parser grammar OmegaConfGrammarParser; +options {tokenVocab = OmegaConfGrammarLexer;} + +// Main rules used to parse OmegaConf strings. + +configValue: text EOF; +singleElement: element EOF; + + +// Composite text expression (may contain interpolations). + +text: (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+; + + +// Elements. + +element: + primitive + | quotedValue + | listContainer + | dictContainer +; + + +// Data structures. + +listContainer: BRACKET_OPEN sequence? BRACKET_CLOSE; // [], [1,2,3], [a,b,[1,2]] +dictContainer: BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE; // {}, {a:10,b:20} +dictKeyValuePair: dictKey COLON element; +sequence: (element (COMMA element?)*) | (COMMA element?)+; + + +// Interpolations. + +interpolation: interpolationNode | interpolationResolver; + +interpolationNode: + INTER_OPEN + DOT* // relative interpolation? + (configKey | BRACKET_OPEN configKey BRACKET_CLOSE) // foo, [foo] + (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)* // .foo, [foo], .foo[bar], [foo].bar[baz] + INTER_CLOSE; +interpolationResolver: INTER_OPEN resolverName COLON sequence? BRACE_CLOSE; +configKey: interpolation | ID | INTER_KEY; +resolverName: (interpolation | ID) (DOT (interpolation | ID))* ; // oc.env, myfunc, ns.${x}, ns1.ns2.f + + +// Primitive types. + +// Ex: "hello world", 'hello ${world}' +quotedValue: (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE; + +primitive: + ( ID // foo_10 + | NULL // null, NULL + | INT // 0, 10, -20, 1_000_000 + | FLOAT // 3.14, -20.0, 1e-1, -10e3 + | BOOL // true, TrUe, false, False + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | + | COLON // : + | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, + | WS // whitespaces + | interpolation + )+; + +// Same as `primitive` except that `COLON` and interpolations are not allowed. +dictKey: + ( ID // foo_10 + | NULL // null, NULL + | INT // 0, 10, -20, 1_000_000 + | FLOAT // 3.14, -20.0, 1e-1, -10e3 + | BOOL // true, TrUe, false, False + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | + | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, + | WS // whitespaces + )+; \ No newline at end of file diff --git a/omegaconf/grammar/__init__.py b/omegaconf/grammar/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/omegaconf/grammar/gen/__init__.py b/omegaconf/grammar/gen/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/omegaconf/grammar_parser.py b/omegaconf/grammar_parser.py new file mode 100644 index 000000000..72d5d7cca --- /dev/null +++ b/omegaconf/grammar_parser.py @@ -0,0 +1,140 @@ +import re +import threading +from typing import Any + +from antlr4 import CommonTokenStream, InputStream, ParserRuleContext +from antlr4.error.ErrorListener import ErrorListener + +from .errors import GrammarParseError + +# Import from visitor in order to check the presence of generated grammar files +# files in a single place. +from .grammar_visitor import ( # type: ignore + OmegaConfGrammarLexer, + OmegaConfGrammarParser, +) + +# Used to cache grammar objects to avoid re-creating them on each call to `parse()`. +# We use a per-thread cache to make it thread-safe. +_grammar_cache = threading.local() + +# Build regex pattern to efficiently identify typical interpolations. +# See test `test_match_simple_interpolation_pattern` for examples. +_config_key = r"[$\w]+" # foo, $0, $bar, $foo_$bar123$ +_key_maybe_brackets = f"{_config_key}|\\[{_config_key}\\]" # foo, [foo], [$bar] +_node_access = f"\\.{_key_maybe_brackets}" # .foo, [foo], [$bar] +_node_path = f"(\\.)*({_key_maybe_brackets})({_node_access})*" # [foo].bar, .foo[bar] +_node_inter = f"\\${{\\s*{_node_path}\\s*}}" # node interpolation ${foo.bar} +_id = "[a-zA-Z_][\\w\\-]*" # foo, foo_bar, foo-bar, abc123 +_resolver_name = f"({_id}(\\.{_id})*)?" # foo, ns.bar3, ns_1.ns_2.b0z +_arg = r"[a-zA-Z_0-9/\-\+.$%*@?|]+" # string representing a resolver argument +_args = f"{_arg}(\\s*,\\s*{_arg})*" # list of resolver arguments +_resolver_inter = f"\\${{\\s*{_resolver_name}\\s*:\\s*{_args}?\\s*}}" # ${foo:bar} +_inter = f"({_node_inter}|{_resolver_inter})" # any kind of interpolation +_outer = "([^$]|\\$(?!{))+" # any character except $ (unless not followed by {) +SIMPLE_INTERPOLATION_PATTERN = re.compile(f"({_outer})?({_inter}({_outer})?)+$", flags=re.ASCII) +# NOTE: SIMPLE_INTERPOLATION_PATTERN must not generate false positive matches: +# it must not accept anything that isn't a valid interpolation (per the +# interpolation grammar defined in `omegaconf/grammar/*.g4`). + + +class OmegaConfErrorListener(ErrorListener): # type: ignore + def syntaxError( + self, + recognizer: Any, + offending_symbol: Any, + line: Any, + column: Any, + msg: Any, + e: Any, + ) -> None: + raise GrammarParseError(str(e) if msg is None else msg) from e + + def reportAmbiguity( + self, + recognizer: Any, + dfa: Any, + startIndex: Any, + stopIndex: Any, + exact: Any, + ambigAlts: Any, + configs: Any, + ) -> None: + raise GrammarParseError("ANTLR error: Ambiguity") # pragma: no cover + + def reportAttemptingFullContext( + self, + recognizer: Any, + dfa: Any, + startIndex: Any, + stopIndex: Any, + conflictingAlts: Any, + configs: Any, + ) -> None: + # Note: for now we raise an error to be safe. However this is mostly a + # performance warning, so in the future this may be relaxed if we need + # to change the grammar in such a way that this warning cannot be + # avoided (another option would be to switch to SLL parsing mode). + raise GrammarParseError("ANTLR error: Attempting Full Context") # pragma: no cover + + def reportContextSensitivity( + self, + recognizer: Any, + dfa: Any, + startIndex: Any, + stopIndex: Any, + prediction: Any, + configs: Any, + ) -> None: + raise GrammarParseError("ANTLR error: ContextSensitivity") # pragma: no cover + + +def parse( + value: str, parser_rule: str = "configValue", lexer_mode: str = "DEFAULT_MODE" +) -> ParserRuleContext: + """ + Parse interpolated string `value` (and return the parse tree). + """ + l_mode = getattr(OmegaConfGrammarLexer, lexer_mode) + istream = InputStream(value) + + cached = getattr(_grammar_cache, "data", None) + if cached is None: + error_listener = OmegaConfErrorListener() + lexer = OmegaConfGrammarLexer(istream) + lexer.removeErrorListeners() + lexer.addErrorListener(error_listener) + lexer.mode(l_mode) + token_stream = CommonTokenStream(lexer) + parser = OmegaConfGrammarParser(token_stream) + parser.removeErrorListeners() + parser.addErrorListener(error_listener) + + # The two lines below could be enabled in the future if we decide to switch + # to SLL prediction mode. Warning though, it has not been fully tested yet! + # from antlr4 import PredictionMode + # parser._interp.predictionMode = PredictionMode.SLL + + # Note that although the input stream `istream` is implicitly cached within + # the lexer, it will be replaced by a new input next time the lexer is re-used. + _grammar_cache.data = lexer, token_stream, parser + + else: + lexer, token_stream, parser = cached + # Replace the old input stream with the new one. + lexer.inputStream = istream + # Initialize the lexer / token stream / parser to process the new input. + lexer.mode(l_mode) + token_stream.setTokenSource(lexer) + parser.reset() + + try: + return getattr(parser, parser_rule)() + except Exception as exc: + if type(exc) is Exception and str(exc) == "Empty Stack": + # This exception is raised by antlr when trying to pop a mode while + # no mode has been pushed. We convert it into an `GrammarParseError` + # to facilitate exception handling from the caller. + raise GrammarParseError("Empty Stack") + else: + raise diff --git a/omegaconf/grammar_visitor.py b/omegaconf/grammar_visitor.py new file mode 100644 index 000000000..3c732507f --- /dev/null +++ b/omegaconf/grammar_visitor.py @@ -0,0 +1,362 @@ +import sys +import warnings +from itertools import zip_longest +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Optional, + Set, + Tuple, + Union, +) + +from antlr4 import TerminalNode + +from .errors import InterpolationResolutionError + +if TYPE_CHECKING: + from .base import Node # noqa F401 + +try: + from omegaconf.grammar.gen.OmegaConfGrammarLexer import OmegaConfGrammarLexer + from omegaconf.grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser + from omegaconf.grammar.gen.OmegaConfGrammarParserVisitor import ( + OmegaConfGrammarParserVisitor, + ) + +except ModuleNotFoundError: # pragma: no cover + print( + "Error importing OmegaConf's generated parsers, run `python setup.py antlr` to regenerate.", + file=sys.stderr, + ) + sys.exit(1) + + +class GrammarVisitor(OmegaConfGrammarParserVisitor): + def __init__( + self, + node_interpolation_callback: Callable[ + [str, Optional[Set[int]]], + Optional["Node"], + ], + resolver_interpolation_callback: Callable[..., Any], + memo: Optional[Set[int]], + **kw: Dict[Any, Any], + ): + """ + Constructor. + + :param node_interpolation_callback: Callback function that is called when + needing to resolve a node interpolation. This function should take a single + string input which is the key's dot path (ex: `"foo.bar"`). + + :param resolver_interpolation_callback: Callback function that is called when + needing to resolve a resolver interpolation. This function should accept + three keyword arguments: `name` (str, the name of the resolver), + `args` (tuple, the inputs to the resolver), and `args_str` (tuple, + the string representation of the inputs to the resolver). + + :param kw: Additional keyword arguments to be forwarded to parent class. + """ + super().__init__(**kw) + self.node_interpolation_callback = node_interpolation_callback + self.resolver_interpolation_callback = resolver_interpolation_callback + self.memo = memo + + def aggregateResult(self, aggregate: List[Any], nextResult: Any) -> List[Any]: + raise NotImplementedError + + def defaultResult(self) -> List[Any]: + # Raising an exception because not currently used (like `aggregateResult()`). + raise NotImplementedError + + def visitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext) -> str: + from ._utils import _get_value + + # interpolation | ID | INTER_KEY + assert ctx.getChildCount() == 1 + child = ctx.getChild(0) + if isinstance(child, OmegaConfGrammarParser.InterpolationContext): + res = _get_value(self.visitInterpolation(child)) + if not isinstance(res, str): + raise InterpolationResolutionError( + f"The following interpolation is used to denote a config key and " + f"thus should return a string, but instead returned `{res}` of " + f"type `{type(res)}`: {ctx.getChild(0).getText()}" + ) + return res + else: + assert isinstance(child, TerminalNode) and isinstance(child.symbol.text, str) + return child.symbol.text + + def visitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext) -> Any: + # text EOF + assert ctx.getChildCount() == 2 + return self.visit(ctx.getChild(0)) + + def visitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext) -> Any: + return self._createPrimitive(ctx) + + def visitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext) -> Dict[Any, Any]: + # BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE + assert ctx.getChildCount() >= 2 + return dict(self.visitDictKeyValuePair(ctx.getChild(i)) for i in range(1, ctx.getChildCount() - 1, 2)) + + def visitElement(self, ctx: OmegaConfGrammarParser.ElementContext) -> Any: + # primitive | quotedValue | listContainer | dictContainer + assert ctx.getChildCount() == 1 + return self.visit(ctx.getChild(0)) + + def visitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext) -> Any: + assert ctx.getChildCount() == 1 # interpolationNode | interpolationResolver + return self.visit(ctx.getChild(0)) + + def visitInterpolationNode( + self, ctx: OmegaConfGrammarParser.InterpolationNodeContext + ) -> Optional["Node"]: + # INTER_OPEN + # DOT* // relative interpolation? + # (configKey | BRACKET_OPEN configKey BRACKET_CLOSE) // foo, [foo] + # (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)* // .foo, [foo], .foo[bar], [foo].bar[baz] + # INTER_CLOSE; + + assert ctx.getChildCount() >= 3 + + inter_key_tokens = [] # parsed elements of the dot path + for child in ctx.getChildren(): + if isinstance(child, TerminalNode): + s = child.symbol + if s.type in [ + OmegaConfGrammarLexer.DOT, + OmegaConfGrammarLexer.BRACKET_OPEN, + OmegaConfGrammarLexer.BRACKET_CLOSE, + ]: + inter_key_tokens.append(s.text) + else: + assert s.type in ( + OmegaConfGrammarLexer.INTER_OPEN, + OmegaConfGrammarLexer.INTER_CLOSE, + ) + else: + assert isinstance(child, OmegaConfGrammarParser.ConfigKeyContext) + inter_key_tokens.append(self.visitConfigKey(child)) + + inter_key = "".join(inter_key_tokens) + return self.node_interpolation_callback(inter_key, self.memo) + + def visitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext) -> Any: + # INTER_OPEN resolverName COLON sequence? BRACE_CLOSE + assert 4 <= ctx.getChildCount() <= 5 + + resolver_name = self.visit(ctx.getChild(1)) + maybe_seq = ctx.getChild(3) + args = [] + args_str = [] + if isinstance(maybe_seq, TerminalNode): # means there are no args + assert maybe_seq.symbol.type == OmegaConfGrammarLexer.BRACE_CLOSE + else: + assert isinstance(maybe_seq, OmegaConfGrammarParser.SequenceContext) + for val, txt in self.visitSequence(maybe_seq): + args.append(val) + args_str.append(txt) + + return self.resolver_interpolation_callback( + name=resolver_name, + args=tuple(args), + args_str=tuple(args_str), + ) + + def visitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext) -> Tuple[Any, Any]: + from ._utils import _get_value + + assert ctx.getChildCount() == 3 # dictKey COLON element + key = self.visit(ctx.getChild(0)) + colon = ctx.getChild(1) + assert isinstance(colon, TerminalNode) and colon.symbol.type == OmegaConfGrammarLexer.COLON + value = _get_value(self.visitElement(ctx.getChild(2))) + return key, value + + def visitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext) -> List[Any]: + # BRACKET_OPEN sequence? BRACKET_CLOSE; + assert ctx.getChildCount() in (2, 3) + if ctx.getChildCount() == 2: + return [] + sequence = ctx.getChild(1) + assert isinstance(sequence, OmegaConfGrammarParser.SequenceContext) + return list(val for val, _ in self.visitSequence(sequence)) # ignore raw text + + def visitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext) -> Any: + return self._createPrimitive(ctx) + + def visitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext) -> str: + # (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE + n = ctx.getChildCount() + assert n in [2, 3] + return str(self.visit(ctx.getChild(1))) if n == 3 else "" + + def visitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext) -> str: + from ._utils import _get_value + + # (interpolation | ID) (DOT (interpolation | ID))* + assert ctx.getChildCount() >= 1 + items = [] + for child in list(ctx.getChildren())[::2]: + if isinstance(child, TerminalNode): + assert child.symbol.type == OmegaConfGrammarLexer.ID + items.append(child.symbol.text) + else: + assert isinstance(child, OmegaConfGrammarParser.InterpolationContext) + item = _get_value(self.visitInterpolation(child)) + if not isinstance(item, str): + raise InterpolationResolutionError( + f"The name of a resolver must be a string, but the interpolation " + f"{child.getText()} resolved to `{item}` which is of type " + f"{type(item)}" + ) + items.append(item) + return ".".join(items) + + def visitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext) -> Generator[Any, None, None]: + from ._utils import _get_value + + # (element (COMMA element?)*) | (COMMA element?)+ + assert ctx.getChildCount() >= 1 + + # DEPRECATED: remove in 2.2 (revert #571) + def empty_str_warning() -> None: + txt = ctx.getText() + warnings.warn( + f"In the sequence `{txt}` some elements are missing: please replace " + f"them with empty quoted strings. " + f"See https://github.com/omry/omegaconf/issues/572 for details.", + category=UserWarning, + ) + + is_previous_comma = True # whether previous child was a comma (init to True) + for child in ctx.getChildren(): + if isinstance(child, OmegaConfGrammarParser.ElementContext): + # Also preserve the original text representation of `child` so + # as to allow backward compatibility with old resolvers (registered + # with `legacy_register_resolver()`). Note that we cannot just cast + # the value to string later as for instance `null` would become "None". + yield _get_value(self.visitElement(child)), child.getText() + is_previous_comma = False + else: + assert isinstance(child, TerminalNode) and child.symbol.type == OmegaConfGrammarLexer.COMMA + if is_previous_comma: + empty_str_warning() + yield "", "" + else: + is_previous_comma = True + if is_previous_comma: + # Trailing comma. + empty_str_warning() + yield "", "" + + def visitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext) -> Any: + # element EOF + assert ctx.getChildCount() == 2 + return self.visit(ctx.getChild(0)) + + def visitText(self, ctx: OmegaConfGrammarParser.TextContext) -> Any: + # (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+ + + # Single interpolation? If yes, return its resolved value "as is". + if ctx.getChildCount() == 1: + c = ctx.getChild(0) + if isinstance(c, OmegaConfGrammarParser.InterpolationContext): + return self.visitInterpolation(c) + + # Otherwise, concatenate string representations together. + return self._unescape(list(ctx.getChildren())) + + def _createPrimitive( + self, + ctx: Union[ + OmegaConfGrammarParser.PrimitiveContext, + OmegaConfGrammarParser.DictKeyContext, + ], + ) -> Any: + # (ID | NULL | INT | FLOAT | BOOL | UNQUOTED_CHAR | COLON | ESC | WS | interpolation)+ + if ctx.getChildCount() == 1: + child = ctx.getChild(0) + if isinstance(child, OmegaConfGrammarParser.InterpolationContext): + return self.visitInterpolation(child) + assert isinstance(child, TerminalNode) + symbol = child.symbol + # Parse primitive types. + if symbol.type in ( + OmegaConfGrammarLexer.ID, + OmegaConfGrammarLexer.UNQUOTED_CHAR, + OmegaConfGrammarLexer.COLON, + ): + return symbol.text + elif symbol.type == OmegaConfGrammarLexer.NULL: + return None + elif symbol.type == OmegaConfGrammarLexer.INT: + return int(symbol.text) + elif symbol.type == OmegaConfGrammarLexer.FLOAT: + return float(symbol.text) + elif symbol.type == OmegaConfGrammarLexer.BOOL: + return symbol.text.lower() == "true" + elif symbol.type == OmegaConfGrammarLexer.ESC: + return self._unescape([child]) + elif symbol.type == OmegaConfGrammarLexer.WS: # pragma: no cover + # A single WS should have been "consumed" by another token. + raise AssertionError("WS should never be reached") + assert False, symbol.type + # Concatenation of multiple items ==> un-escape the concatenation. + return self._unescape(list(ctx.getChildren())) + + def _unescape( + self, + seq: List[Union[TerminalNode, OmegaConfGrammarParser.InterpolationContext]], + ) -> str: + """ + Concatenate all symbols / interpolations in `seq`, unescaping symbols as needed. + + Interpolations are resolved and cast to string *WITHOUT* escaping their result + (it is assumed that whatever escaping is required was already handled during the + resolving of the interpolation). + """ + chrs = [] + for node, next_node in zip_longest(seq, seq[1:]): + if isinstance(node, TerminalNode): + s = node.symbol + if s.type == OmegaConfGrammarLexer.ESC_INTER: + # `ESC_INTER` is of the form `\\...\${`: the formula below computes + # the number of characters to keep at the end of the string to remove + # the correct number of backslashes. + text = s.text[-(len(s.text) // 2 + 1) :] + elif ( + # Character sequence identified as requiring un-escaping. + s.type == OmegaConfGrammarLexer.ESC + or ( + # At top level, we need to un-escape backslashes that precede + # an interpolation. + s.type == OmegaConfGrammarLexer.TOP_ESC + and isinstance(next_node, OmegaConfGrammarParser.InterpolationContext) + ) + or ( + # In a quoted sring, we need to un-escape backslashes that + # either end the string, or are followed by an interpolation. + s.type == OmegaConfGrammarLexer.QUOTED_ESC + and ( + next_node is None + or isinstance(next_node, OmegaConfGrammarParser.InterpolationContext) + ) + ) + ): + text = s.text[1::2] # un-escape the sequence + else: + text = s.text # keep the original text + else: + assert isinstance(node, OmegaConfGrammarParser.InterpolationContext) + text = str(self.visitInterpolation(node)) + chrs.append(text) + + return "".join(chrs) diff --git a/omegaconf/listconfig.py b/omegaconf/listconfig.py new file mode 100644 index 000000000..b0f9a4422 --- /dev/null +++ b/omegaconf/listconfig.py @@ -0,0 +1,643 @@ +import copy +import itertools +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + MutableSequence, + Optional, + Tuple, + Type, + Union, +) + +from ._utils import ( + ValueKind, + _is_missing_literal, + _is_none, + _resolve_optional, + format_and_raise, + get_value_kind, + is_int, + is_primitive_list, + is_structured_config, + type_str, +) +from .base import Box, ContainerMetadata, Node +from .basecontainer import BaseContainer +from .errors import ( + ConfigAttributeError, + ConfigTypeError, + ConfigValueError, + KeyValidationError, + MissingMandatoryValue, + ReadonlyConfigError, + ValidationError, +) + + +class ListConfig(BaseContainer, MutableSequence[Any]): + _content: Union[List[Node], None, str] + + def __init__( + self, + content: Union[List[Any], Tuple[Any, ...], "ListConfig", str, None], + key: Any = None, + parent: Optional[Box] = None, + element_type: Union[Type[Any], Any] = Any, + is_optional: bool = True, + ref_type: Union[Type[Any], Any] = Any, + flags: Optional[Dict[str, bool]] = None, + ) -> None: + try: + if isinstance(content, ListConfig): + if flags is None: + flags = content._metadata.flags + super().__init__( + parent=parent, + metadata=ContainerMetadata( + ref_type=ref_type, + object_type=list, + key=key, + optional=is_optional, + element_type=element_type, + key_type=int, + flags=flags, + ), + ) + + if isinstance(content, ListConfig): + metadata = copy.deepcopy(content._metadata) + metadata.key = key + metadata.ref_type = ref_type + metadata.optional = is_optional + metadata.element_type = element_type + self.__dict__["_metadata"] = metadata + self._set_value(value=content, flags=flags) + except Exception as ex: + format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex)) + + def _validate_get(self, key: Any, value: Any = None) -> None: + if not isinstance(key, (int, slice)): + raise KeyValidationError("ListConfig indices must be integers or slices, not $KEY_TYPE") + + def _validate_set(self, key: Any, value: Any) -> None: + from omegaconf import OmegaConf + + self._validate_get(key, value) + + if self._get_flag("readonly"): + raise ReadonlyConfigError("ListConfig is read-only") + + if 0 <= key < self.__len__(): + target = self._get_node(key) + if target is not None: + assert isinstance(target, Node) + if value is None and not target._is_optional(): + raise ValidationError("$FULL_KEY is not optional and cannot be assigned None") + + vk = get_value_kind(value) + if vk == ValueKind.MANDATORY_MISSING: + return + else: + is_optional, target_type = _resolve_optional(self._metadata.element_type) + value_type = OmegaConf.get_type(value) + + if (value_type is None and not is_optional) or ( + is_structured_config(target_type) + and value_type is not None + and not issubclass(value_type, target_type) + ): + msg = ( + f"Invalid type assigned: {type_str(value_type)} is not a " + f"subclass of {type_str(target_type)}. value: {value}" + ) + raise ValidationError(msg) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "ListConfig": + res = ListConfig(None) + res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo) + res.__dict__["_flags_cache"] = copy.deepcopy(self.__dict__["_flags_cache"], memo=memo) + + src_content = self.__dict__["_content"] + if isinstance(src_content, list): + content_copy: List[Optional[Node]] = [] + for v in src_content: + old_parent = v.__dict__["_parent"] + try: + v.__dict__["_parent"] = None + vc = copy.deepcopy(v, memo=memo) + vc.__dict__["_parent"] = res + content_copy.append(vc) + finally: + v.__dict__["_parent"] = old_parent + else: + # None and strings can be assigned as is + content_copy = src_content + + res.__dict__["_content"] = content_copy + res.__dict__["_parent"] = self.__dict__["_parent"] + + return res + + def copy(self) -> "ListConfig": + return copy.copy(self) + + # hide content while inspecting in debugger + def __dir__(self) -> Iterable[str]: + if self._is_missing() or self._is_none(): + return [] + return [str(x) for x in range(0, len(self))] + + def __setattr__(self, key: str, value: Any) -> None: + self._format_and_raise( + key=key, + value=value, + cause=ConfigAttributeError("ListConfig does not support attribute access"), + ) + assert False + + def __getattr__(self, key: str) -> Any: + # PyCharm is sometimes inspecting __members__, be sure to tell it we don't have that. + if key == "__members__": + raise AttributeError() + + if key == "__name__": + raise AttributeError() + + if is_int(key): + return self.__getitem__(int(key)) + else: + self._format_and_raise( + key=key, + value=None, + cause=ConfigAttributeError("ListConfig does not support attribute access"), + ) + + def __getitem__(self, index: Union[int, slice]) -> Any: + try: + if self._is_missing(): + raise MissingMandatoryValue("ListConfig is missing") + self._validate_get(index, None) + if self._is_none(): + raise TypeError("ListConfig object representing None is not subscriptable") + + assert isinstance(self.__dict__["_content"], list) + if isinstance(index, slice): + result = [] + start, stop, step = self._correct_index_params(index) + for slice_idx in itertools.islice(range(0, len(self)), start, stop, step): + val = self._resolve_with_default( + key=slice_idx, value=self.__dict__["_content"][slice_idx] + ) + result.append(val) + if index.step and index.step < 0: + result.reverse() + return result + else: + return self._resolve_with_default(key=index, value=self.__dict__["_content"][index]) + except Exception as e: + self._format_and_raise(key=index, value=None, cause=e) + + def _correct_index_params(self, index: slice) -> Tuple[int, int, int]: + start = index.start + stop = index.stop + step = index.step + if index.start and index.start < 0: + start = self.__len__() + index.start + if index.stop and index.stop < 0: + stop = self.__len__() + index.stop + if index.step and index.step < 0: + step = abs(step) + if start and stop: + if start > stop: + start, stop = stop + 1, start + 1 + else: + start = stop = 0 + elif not start and stop: + start = list(range(self.__len__() - 1, stop, -step))[0] + stop = None + elif start and not stop: + stop = start + 1 + start = (stop - 1) % step + else: + start = (self.__len__() - 1) % step + return start, stop, step + + def _set_at_index(self, index: Union[int, slice], value: Any) -> None: + self._set_item_impl(index, value) + + def __setitem__(self, index: Union[int, slice], value: Any) -> None: + try: + if isinstance(index, slice): + _ = iter(value) # check iterable + self_indices = index.indices(len(self)) + indexes = range(*self_indices) + + # Ensure lengths match for extended slice assignment + if index.step not in (None, 1): + if len(indexes) != len(value): + raise ValueError( + f"attempt to assign sequence of size {len(value)}" + f" to extended slice of size {len(indexes)}" + ) + + # Initialize insertion offsets for empty slices + if len(indexes) == 0: + curr_index = self_indices[0] - 1 + val_i = -1 + + work_copy = self.copy() # For atomicity manipulate a copy + + # Delete and optionally replace non empty slices + only_removed = 0 + for val_i, i in enumerate(indexes): + curr_index = i - only_removed + del work_copy[curr_index] + if val_i < len(value): + work_copy.insert(curr_index, value[val_i]) + else: + only_removed += 1 + + # Insert any remaining input items + for val_i in range(val_i + 1, len(value)): + curr_index += 1 + work_copy.insert(curr_index, value[val_i]) + + # Reinitialize self with work_copy + self.clear() + self.extend(work_copy) + else: + self._set_at_index(index, value) + except Exception as e: + self._format_and_raise(key=index, value=value, cause=e) + + def append(self, item: Any) -> None: + content = self.__dict__["_content"] + index = len(content) + content.append(None) + try: + self._set_item_impl(index, item) + except Exception as e: + del content[index] + self._format_and_raise(key=index, value=item, cause=e) + assert False + + def _update_keys(self) -> None: + for i in range(len(self)): + node = self._get_node(i) + if node is not None: + assert isinstance(node, Node) + node._metadata.key = i + + def insert(self, index: int, item: Any) -> None: + from omegaconf.omegaconf import _maybe_wrap + + try: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot insert into a read-only ListConfig") + if self._is_none(): + raise TypeError("Cannot insert into ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot insert into missing ListConfig") + + try: + assert isinstance(self.__dict__["_content"], list) + # insert place holder + self.__dict__["_content"].insert(index, None) + is_optional, ref_type = _resolve_optional(self._metadata.element_type) + node = _maybe_wrap( + ref_type=ref_type, + key=index, + value=item, + is_optional=is_optional, + parent=self, + ) + self._validate_set(key=index, value=node) + self._set_at_index(index, node) + self._update_keys() + except Exception: + del self.__dict__["_content"][index] + self._update_keys() + raise + except Exception as e: + self._format_and_raise(key=index, value=item, cause=e) + assert False + + def extend(self, lst: Iterable[Any]) -> None: + assert isinstance(lst, (tuple, list, ListConfig)) + for x in lst: + self.append(x) + + def remove(self, x: Any) -> None: + del self[self.index(x)] + + def __delitem__(self, key: Union[int, slice]) -> None: + if self._get_flag("readonly"): + self._format_and_raise( + key=key, + value=None, + cause=ReadonlyConfigError("Cannot delete item from read-only ListConfig"), + ) + del self.__dict__["_content"][key] + self._update_keys() + + def clear(self) -> None: + del self[:] + + def index(self, x: Any, start: Optional[int] = None, end: Optional[int] = None) -> int: + if start is None: + start = 0 + if end is None: + end = len(self) + assert start >= 0 + assert end <= len(self) + found_idx = -1 + for idx in range(start, end): + item = self[idx] + if x == item: + found_idx = idx + break + if found_idx != -1: + return found_idx + else: + self._format_and_raise( + key=None, + value=None, + cause=ConfigValueError("Item not found in ListConfig"), + ) + assert False + + def count(self, x: Any) -> int: + c = 0 + for item in self: + if item == x: + c = c + 1 + return c + + def _get_node( + self, + key: Union[int, slice], + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + try: + if self._is_none(): + raise TypeError("Cannot get_node from a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot get_node from a missing ListConfig") + assert isinstance(self.__dict__["_content"], list) + if validate_access: + self._validate_get(key) + + value = self.__dict__["_content"][key] + if value is not None: + if isinstance(key, slice): + assert isinstance(value, list) + for v in value: + if throw_on_missing_value and v._is_missing(): + raise MissingMandatoryValue("Missing mandatory value") + else: + assert isinstance(value, Node) + if throw_on_missing_value and value._is_missing(): + raise MissingMandatoryValue("Missing mandatory value: $KEY") + return value + except (IndexError, TypeError, MissingMandatoryValue, KeyValidationError) as e: + if isinstance(e, MissingMandatoryValue) and throw_on_missing_value: + raise + if validate_access: + self._format_and_raise(key=key, value=None, cause=e) + assert False + else: + return None + + def get(self, index: int, default_value: Any = None) -> Any: + try: + if self._is_none(): + raise TypeError("Cannot get from a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot get from a missing ListConfig") + self._validate_get(index, None) + assert isinstance(self.__dict__["_content"], list) + return self._resolve_with_default( + key=index, + value=self.__dict__["_content"][index], + default_value=default_value, + ) + except Exception as e: + self._format_and_raise(key=index, value=None, cause=e) + assert False + + def pop(self, index: int = -1) -> Any: + try: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot pop from read-only ListConfig") + if self._is_none(): + raise TypeError("Cannot pop from a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot pop from a missing ListConfig") + + assert isinstance(self.__dict__["_content"], list) + node = self._get_child(index) + assert isinstance(node, Node) + ret = self._resolve_with_default(key=index, value=node, default_value=None) + del self.__dict__["_content"][index] + self._update_keys() + return ret + except KeyValidationError as e: + self._format_and_raise(key=index, value=None, cause=e, type_override=ConfigTypeError) + assert False + except Exception as e: + self._format_and_raise(key=index, value=None, cause=e) + assert False + + def sort(self, key: Optional[Callable[[Any], Any]] = None, reverse: bool = False) -> None: + try: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot sort a read-only ListConfig") + if self._is_none(): + raise TypeError("Cannot sort a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot sort a missing ListConfig") + + if key is None: + + def key1(x: Any) -> Any: + return x._value() + + else: + + def key1(x: Any) -> Any: + return key(x._value()) # type: ignore + + assert isinstance(self.__dict__["_content"], list) + self.__dict__["_content"].sort(key=key1, reverse=reverse) + + except Exception as e: + self._format_and_raise(key=None, value=None, cause=e) + assert False + + def __eq__(self, other: Any) -> bool: + if isinstance(other, (list, tuple)) or other is None: + other = ListConfig(other, flags={"allow_objects": True}) + return ListConfig._list_eq(self, other) + if other is None or isinstance(other, ListConfig): + return ListConfig._list_eq(self, other) + if self._is_missing(): + return _is_missing_literal(other) + return NotImplemented + + def __ne__(self, other: Any) -> bool: + x = self.__eq__(other) + if x is not NotImplemented: + return not x + return NotImplemented + + def __hash__(self) -> int: + return hash(str(self)) + + def __iter__(self) -> Iterator[Any]: + return self._iter_ex(resolve=True) + + class ListIterator(Iterator[Any]): + def __init__(self, lst: Any, resolve: bool) -> None: + self.resolve = resolve + self.iterator = iter(lst.__dict__["_content"]) + self.index = 0 + from .nodes import ValueNode + + self.ValueNode = ValueNode + + def __next__(self) -> Any: + x = next(self.iterator) + if self.resolve: + x = x._dereference_node() + if x._is_missing(): + raise MissingMandatoryValue(f"Missing value at index {self.index}") + + self.index = self.index + 1 + if isinstance(x, self.ValueNode): + return x._value() + else: + # Must be omegaconf.Container. not checking for perf reasons. + if x._is_none(): + return None + return x + + def __repr__(self) -> str: # pragma: no cover + return f"ListConfig.ListIterator(resolve={self.resolve})" + + def _iter_ex(self, resolve: bool) -> Iterator[Any]: + try: + if self._is_none(): + raise TypeError("Cannot iterate a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot iterate a missing ListConfig") + + return ListConfig.ListIterator(self, resolve) + except (TypeError, MissingMandatoryValue) as e: + self._format_and_raise(key=None, value=None, cause=e) + assert False + + def __add__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig": + # res is sharing this list's parent to allow interpolation to work as expected + res = ListConfig(parent=self._get_parent(), content=[]) + res.extend(self) + res.extend(other) + return res + + def __radd__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig": + # res is sharing this list's parent to allow interpolation to work as expected + res = ListConfig(parent=self._get_parent(), content=[]) + res.extend(other) + res.extend(self) + return res + + def __iadd__(self, other: Iterable[Any]) -> "ListConfig": + self.extend(other) + return self + + def __contains__(self, item: Any) -> bool: + if self._is_none(): + raise TypeError("Cannot check if an item is in a ListConfig object representing None") + if self._is_missing(): + raise MissingMandatoryValue("Cannot check if an item is in missing ListConfig") + + lst = self.__dict__["_content"] + for x in lst: + x = x._dereference_node() + if x == item: + return True + return False + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + try: + previous_content = self.__dict__["_content"] + previous_metadata = self.__dict__["_metadata"] + self._set_value_impl(value, flags) + except Exception as e: + self.__dict__["_content"] = previous_content + self.__dict__["_metadata"] = previous_metadata + raise e + + def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + from omegaconf import MISSING, flag_override + + if flags is None: + flags = {} + + vk = get_value_kind(value, strict_interpolation_validation=True) + if _is_none(value): + if not self._is_optional(): + raise ValidationError("Non optional ListConfig cannot be constructed from None") + self.__dict__["_content"] = None + self._metadata.object_type = None + elif vk is ValueKind.MANDATORY_MISSING: + self.__dict__["_content"] = MISSING + self._metadata.object_type = None + elif vk == ValueKind.INTERPOLATION: + self.__dict__["_content"] = value + self._metadata.object_type = None + else: + if not (is_primitive_list(value) or isinstance(value, ListConfig)): + type_ = type(value) + msg = f"Invalid value assigned: {type_.__name__} is not a ListConfig, list or tuple." + raise ValidationError(msg) + + self.__dict__["_content"] = [] + if isinstance(value, ListConfig): + self._metadata.flags = copy.deepcopy(flags) + # disable struct and readonly for the construction phase + # retaining other flags like allow_objects. The real flags are restored at the end of this function + with flag_override(self, ["struct", "readonly"], False): + for item in value._iter_ex(resolve=False): + self.append(item) + elif is_primitive_list(value): + with flag_override(self, ["struct", "readonly"], False): + for item in value: + self.append(item) + self._metadata.object_type = list + + @staticmethod + def _list_eq(l1: Optional["ListConfig"], l2: Optional["ListConfig"]) -> bool: + l1_none = l1.__dict__["_content"] is None + l2_none = l2.__dict__["_content"] is None + if l1_none and l2_none: + return True + if l1_none != l2_none: + return False + + assert isinstance(l1, ListConfig) + assert isinstance(l2, ListConfig) + if len(l1) != len(l2): + return False + for i in range(len(l1)): + if not BaseContainer._item_eq(l1, i, l2, i): + return False + + return True diff --git a/omegaconf/nodes.py b/omegaconf/nodes.py new file mode 100644 index 000000000..44ce6a591 --- /dev/null +++ b/omegaconf/nodes.py @@ -0,0 +1,513 @@ +import copy +import math +import sys +from abc import abstractmethod +from enum import Enum +from pathlib import Path +from typing import Any, Dict, Optional, Type, Union + +from omegaconf._utils import ( + ValueKind, + _is_interpolation, + get_type_of, + get_value_kind, + is_primitive_container, + type_str, +) +from omegaconf.base import Box, DictKeyType, Metadata, Node +from omegaconf.errors import ReadonlyConfigError, UnsupportedValueType, ValidationError + + +class ValueNode(Node): + _val: Any + + def __init__(self, parent: Optional[Box], value: Any, metadata: Metadata): + from omegaconf import read_write + + super().__init__(parent=parent, metadata=metadata) + with read_write(self): + self._set_value(value) # lgtm [py/init-calls-subclass] + + def _value(self) -> Any: + return self._val + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot set value of read-only config node") + + if isinstance(value, str) and get_value_kind(value, strict_interpolation_validation=True) in ( + ValueKind.INTERPOLATION, + ValueKind.MANDATORY_MISSING, + ): + self._val = value + else: + self._val = self.validate_and_convert(value) + + def _strict_validate_type(self, value: Any) -> None: + ref_type = self._metadata.ref_type + if isinstance(ref_type, type) and type(value) is not ref_type: + type_hint = type_str(self._metadata.type_hint) + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" + ) + + def validate_and_convert(self, value: Any) -> Any: + """ + Validates input and converts to canonical form + :param value: input value + :return: converted value ("100" may be converted to 100 for example) + """ + if value is None: + if self._is_optional(): + return None + ref_type_str = type_str(self._metadata.ref_type) + raise ValidationError(f"Incompatible value '{value}' for field of type '{ref_type_str}'") + + # Subclasses can assume that `value` is not None in + # `_validate_and_convert_impl()` and in `_strict_validate_type()`. + if self._get_flag("convert") is False: + self._strict_validate_type(value) + return value + else: + return self._validate_and_convert_impl(value) + + @abstractmethod + def _validate_and_convert_impl(self, value: Any) -> Any: + ... + + def __str__(self) -> str: + return str(self._val) + + def __repr__(self) -> str: + return repr(self._val) if hasattr(self, "_val") else "__INVALID__" + + def __eq__(self, other: Any) -> bool: + if isinstance(other, AnyNode): + return self._val == other._val # type: ignore + else: + return self._val == other # type: ignore + + def __ne__(self, other: Any) -> bool: + x = self.__eq__(other) + assert x is not NotImplemented + return not x + + def __hash__(self) -> int: + return hash(self._val) + + def _deepcopy_impl(self, res: Any, memo: Dict[int, Any]) -> None: + res.__dict__["_metadata"] = copy.deepcopy(self._metadata, memo=memo) + # shallow copy for value to support non-copyable value + res.__dict__["_val"] = self._val + + # parent is retained, but not copied + res.__dict__["_parent"] = self._parent + + def _is_optional(self) -> bool: + return self._metadata.optional + + def _is_interpolation(self) -> bool: + return _is_interpolation(self._value()) + + def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: + parent = self._get_parent() + if parent is None: + if self._metadata.key is None: + return "" + else: + return str(self._metadata.key) + else: + return parent._get_full_key(self._metadata.key) + + +class AnyNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata(ref_type=Any, object_type=None, key=key, optional=True, flags=flags), + ) + + def _validate_and_convert_impl(self, value: Any) -> Any: + from ._utils import is_primitive_type_annotation + + # allow_objects is internal and not an official API. use at your own risk. + # Please be aware that this support is subject to change without notice. + # If this is deemed useful and supportable it may become an official API. + + if self._get_flag("allow_objects") is not True and not is_primitive_type_annotation(value): + t = get_type_of(value) + raise UnsupportedValueType(f"Value '{t.__name__}' is not a supported primitive type") + return value + + def __deepcopy__(self, memo: Dict[int, Any]) -> "AnyNode": + res = AnyNode() + self._deepcopy_impl(res, memo) + return res + + +class StringNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=str, + object_type=str, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> str: + from omegaconf import OmegaConf + + if OmegaConf.is_config(value) or is_primitive_container(value) or isinstance(value, bytes): + raise ValidationError("Cannot convert '$VALUE_TYPE' to string: '$VALUE'") + return str(value) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "StringNode": + res = StringNode() + self._deepcopy_impl(res, memo) + return res + + +class PathNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=Path, + object_type=Path, + flags=flags, + ), + ) + + def _strict_validate_type(self, value: Any) -> None: + if not isinstance(value, Path): + raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' is not an instance of 'pathlib.Path'") + + def _validate_and_convert_impl(self, value: Any) -> Path: + if not isinstance(value, (str, Path)): + raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Path") + + return Path(value) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "PathNode": + res = PathNode() + self._deepcopy_impl(res, memo) + return res + + +class IntegerNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=int, + object_type=int, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> int: + try: + if type(value) in (str, int): + val = int(value) + else: + raise ValueError() + except ValueError: + raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Integer") + return val + + def __deepcopy__(self, memo: Dict[int, Any]) -> "IntegerNode": + res = IntegerNode() + self._deepcopy_impl(res, memo) + return res + + +class BytesNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=bytes, + object_type=bytes, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> bytes: + if not isinstance(value, bytes): + raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' is not of type 'bytes'") + return value + + def __deepcopy__(self, memo: Dict[int, Any]) -> "BytesNode": + res = BytesNode() + self._deepcopy_impl(res, memo) + return res + + +class FloatNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=float, + object_type=float, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> float: + try: + if type(value) in (float, str, int): + return float(value) + else: + raise ValueError() + except ValueError: + raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Float") + + def __eq__(self, other: Any) -> bool: + if isinstance(other, ValueNode): + other_val = other._val + else: + other_val = other + if self._val is None and other is None: + return True + if self._val is None and other is not None: + return False + if self._val is not None and other is None: + return False + nan1 = math.isnan(self._val) if isinstance(self._val, float) else False + nan2 = math.isnan(other_val) if isinstance(other_val, float) else False + return self._val == other_val or (nan1 and nan2) + + def __hash__(self) -> int: + return hash(self._val) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "FloatNode": + res = FloatNode() + self._deepcopy_impl(res, memo) + return res + + +class BooleanNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=bool, + object_type=bool, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, int): + return value != 0 + elif isinstance(value, str): + try: + return self._validate_and_convert_impl(int(value)) + except ValueError as e: + if value.lower() in ("yes", "y", "on", "true"): + return True + elif value.lower() in ("no", "n", "off", "false"): + return False + else: + raise ValidationError( + "Value '$VALUE' is not a valid bool (type $VALUE_TYPE)" + ).with_traceback(sys.exc_info()[2]) from e + else: + raise ValidationError("Value '$VALUE' is not a valid bool (type $VALUE_TYPE)") + + def __deepcopy__(self, memo: Dict[int, Any]) -> "BooleanNode": + res = BooleanNode() + self._deepcopy_impl(res, memo) + return res + + +class EnumNode(ValueNode): # lgtm [py/missing-equals] : Intentional. + """ + NOTE: EnumNode is serialized to yaml as a string ("Color.BLUE"), not as a fully qualified yaml type. + this means serialization to YAML of a typed config (with EnumNode) will not retain the type of the Enum + when loaded. + This is intentional, Please open an issue against OmegaConf if you wish to discuss this decision. + """ + + def __init__( + self, + enum_type: Type[Enum], + value: Optional[Union[Enum, str]] = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + if not isinstance(enum_type, type) or not issubclass(enum_type, Enum): + raise ValidationError(f"EnumNode can only operate on Enum subclasses ({enum_type})") + self.fields: Dict[str, str] = {} + self.enum_type: Type[Enum] = enum_type + for name, constant in enum_type.__members__.items(): + self.fields[name] = constant.value + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=enum_type, + object_type=enum_type, + flags=flags, + ), + ) + + def _strict_validate_type(self, value: Any) -> None: + ref_type = self._metadata.ref_type + if not isinstance(value, ref_type): + type_hint = type_str(self._metadata.type_hint) + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" + ) + + def _validate_and_convert_impl(self, value: Any) -> Enum: + return self.validate_and_convert_to_enum(enum_type=self.enum_type, value=value) + + @staticmethod + def validate_and_convert_to_enum(enum_type: Type[Enum], value: Any) -> Enum: + if not isinstance(value, (str, int)) and not isinstance(value, enum_type): + raise ValidationError(f"Value $VALUE ($VALUE_TYPE) is not a valid input for {enum_type}") + + if isinstance(value, enum_type): + return value + + try: + if isinstance(value, (float, bool)): + raise ValueError + + if isinstance(value, int): + return enum_type(value) + + if isinstance(value, str): + prefix = f"{enum_type.__name__}." + if value.startswith(prefix): + value = value[len(prefix) :] + return enum_type[value] + + assert False + + except (ValueError, KeyError) as e: + valid = ", ".join([x for x in enum_type.__members__.keys()]) + raise ValidationError(f"Invalid value '$VALUE', expected one of [{valid}]").with_traceback( + sys.exc_info()[2] + ) from e + + def __deepcopy__(self, memo: Dict[int, Any]) -> "EnumNode": + res = EnumNode(enum_type=self.enum_type) + self._deepcopy_impl(res, memo) + return res + + +class InterpolationResultNode(ValueNode): + """ + Special node type, used to wrap interpolation results. + """ + + def __init__( + self, + value: Any, + key: Any = None, + parent: Optional[Box] = None, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata(ref_type=Any, object_type=None, key=key, optional=True, flags=flags), + ) + # In general we should not try to write into interpolation results. + if flags is None or "readonly" not in flags: + self._set_flag("readonly", True) + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + if self._get_flag("readonly"): + raise ReadonlyConfigError("Cannot set value of read-only config node") + self._val = self.validate_and_convert(value) + + def _validate_and_convert_impl(self, value: Any) -> Any: + # Interpolation results may be anything. + return value + + def __deepcopy__(self, memo: Dict[int, Any]) -> "InterpolationResultNode": + # Currently there should be no need to deep-copy such nodes. + raise NotImplementedError + + def _is_interpolation(self) -> bool: + # The result of an interpolation cannot be itself an interpolation. + return False diff --git a/omegaconf/omegaconf.py b/omegaconf/omegaconf.py new file mode 100644 index 000000000..e438eb244 --- /dev/null +++ b/omegaconf/omegaconf.py @@ -0,0 +1,1144 @@ +"""OmegaConf module""" +import copy +import inspect +import io +import os +import pathlib +import sys +import warnings +from collections import defaultdict +from contextlib import contextmanager +from enum import Enum +from textwrap import dedent +from typing import ( + IO, + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Optional, + Set, + Tuple, + Type, + Union, + overload, +) + +import yaml + +from . import DictConfig, DictKeyType, ListConfig +from ._utils import ( + _DEFAULT_MARKER_, + _ensure_container, + _get_value, + format_and_raise, + get_dict_key_value_types, + get_list_element_type, + get_omega_conf_dumper, + get_type_of, + is_attr_class, + is_dataclass, + is_dict_annotation, + is_int, + is_list_annotation, + is_primitive_container, + is_primitive_dict, + is_primitive_list, + is_structured_config, + is_tuple_annotation, + is_union_annotation, + nullcontext, + split_key, + type_str, +) +from .base import Box, Container, Node, SCMode, UnionNode +from .basecontainer import BaseContainer +from .errors import ( + MissingMandatoryValue, + OmegaConfBaseException, + UnsupportedInterpolationType, + ValidationError, +) +from .nodes import ( + AnyNode, + BooleanNode, + BytesNode, + EnumNode, + FloatNode, + IntegerNode, + PathNode, + StringNode, + ValueNode, +) + +MISSING: Any = "???" + +Resolver = Callable[..., Any] + + +def II(interpolation: str) -> Any: + """ + Equivalent to ``${interpolation}`` + + :param interpolation: + :return: input ``${node}`` with type Any + """ + return "${" + interpolation + "}" + + +def SI(interpolation: str) -> Any: + """ + Use this for String interpolation, for example ``"http://${host}:${port}"`` + + :param interpolation: interpolation string + :return: input interpolation with type ``Any`` + """ + return interpolation + + +def register_default_resolvers() -> None: + from omegaconf.resolvers import oc + + OmegaConf.register_new_resolver("oc.create", oc.create) + OmegaConf.register_new_resolver("oc.decode", oc.decode) + OmegaConf.register_new_resolver("oc.deprecated", oc.deprecated) + OmegaConf.register_new_resolver("oc.env", oc.env) + OmegaConf.register_new_resolver("oc.select", oc.select) + OmegaConf.register_new_resolver("oc.dict.keys", oc.dict.keys) + OmegaConf.register_new_resolver("oc.dict.values", oc.dict.values) + + +class OmegaConf: + """OmegaConf primary class""" + + def __init__(self) -> None: + raise NotImplementedError("Use one of the static construction functions") + + @staticmethod + def structured( + obj: Any, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> Any: + return OmegaConf.create(obj, parent, flags) + + @staticmethod + @overload + def create( + obj: str, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> Union[DictConfig, ListConfig]: + ... + + @staticmethod + @overload + def create( + obj: Union[List[Any], Tuple[Any, ...]], + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> ListConfig: + ... + + @staticmethod + @overload + def create( + obj: DictConfig, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> DictConfig: + ... + + @staticmethod + @overload + def create( + obj: ListConfig, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> ListConfig: + ... + + @staticmethod + @overload + def create( + obj: Optional[Dict[Any, Any]] = None, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> DictConfig: + ... + + @staticmethod + def create( # noqa F811 + obj: Any = _DEFAULT_MARKER_, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> Union[DictConfig, ListConfig]: + return OmegaConf._create_impl( + obj=obj, + parent=parent, + flags=flags, + ) + + @staticmethod + def load(file_: Union[str, pathlib.Path, IO[Any]]) -> Union[DictConfig, ListConfig]: + from ._utils import get_yaml_loader + + if isinstance(file_, (str, pathlib.Path)): + with io.open(os.path.abspath(file_), "r", encoding="utf-8") as f: + obj = yaml.load(f, Loader=get_yaml_loader()) + elif getattr(file_, "read", None): + obj = yaml.load(file_, Loader=get_yaml_loader()) + else: + raise TypeError("Unexpected file type") + + if obj is not None and not isinstance(obj, (list, dict, str)): + raise IOError(f"Invalid loaded object type: {type(obj).__name__}") # pragma: no cover + + ret: Union[DictConfig, ListConfig] + if obj is None: + ret = OmegaConf.create() + else: + ret = OmegaConf.create(obj) + return ret + + @staticmethod + def save(config: Any, f: Union[str, pathlib.Path, IO[Any]], resolve: bool = False) -> None: + """ + Save as configuration object to a file + + :param config: omegaconf.Config object (DictConfig or ListConfig). + :param f: filename or file object + :param resolve: True to save a resolved config (defaults to False) + """ + if is_dataclass(config) or is_attr_class(config): + config = OmegaConf.create(config) + data = OmegaConf.to_yaml(config, resolve=resolve) + if isinstance(f, (str, pathlib.Path)): + with io.open(os.path.abspath(f), "w", encoding="utf-8") as file: + file.write(data) + elif hasattr(f, "write"): + f.write(data) + f.flush() + else: + raise TypeError("Unexpected file type") + + @staticmethod + def from_cli(args_list: Optional[List[str]] = None) -> DictConfig: + if args_list is None: + # Skip program name + args_list = sys.argv[1:] + return OmegaConf.from_dotlist(args_list) + + @staticmethod + def from_dotlist(dotlist: List[str]) -> DictConfig: + """ + Creates config from the content sys.argv or from the specified args list of not None + + :param dotlist: A list of dotlist-style strings, e.g. ``["foo.bar=1", "baz=qux"]``. + :return: A ``DictConfig`` object created from the dotlist. + """ + conf = OmegaConf.create() + conf.merge_with_dotlist(dotlist) + return conf + + @staticmethod + def merge( + *configs: Union[ + DictConfig, + ListConfig, + Dict[DictKeyType, Any], + List[Any], + Tuple[Any, ...], + Any, + ], + extend_lists: bool = False, + remove_duplicates: bool = False, + ) -> Union[ListConfig, DictConfig]: + """ + Merge a list of previously created configs into a single one + + :param configs: Input configs + :param extend_lists: flag to deep merge ListConfigs + :param remove_duplicates: lists entries are unique. This flag is ignored unless `extend_lists=True` + :return: the merged config object. + """ + assert len(configs) > 0 + target = copy.deepcopy(configs[0]) + target = _ensure_container(target) + assert isinstance(target, (DictConfig, ListConfig)) + + with flag_override(target, "readonly", False): + target.merge_with( + *configs[1:], + extend_lists=extend_lists, + remove_duplicates=remove_duplicates, + ) + turned_readonly = target._get_flag("readonly") is True + + if turned_readonly: + OmegaConf.set_readonly(target, True) + + return target + + @staticmethod + def unsafe_merge( + *configs: Union[ + DictConfig, + ListConfig, + Dict[DictKeyType, Any], + List[Any], + Tuple[Any, ...], + Any, + ], + extend_lists: bool = False, + remove_duplicates: bool = False, + ) -> Union[ListConfig, DictConfig]: + """ + Merge a list of previously created configs into a single one + This is much faster than OmegaConf.merge() as the input configs are not copied. + However, the input configs must not be used after this operation as will become inconsistent. + + :param configs: Input configs + :param extend_lists: flag to deep merge ListConfigs + :param remove_duplicates: lists entries are unique. This flag is ignored unless `extend_lists=True` + :return: the merged config object. + """ + assert len(configs) > 0 + target = configs[0] + target = _ensure_container(target) + assert isinstance(target, (DictConfig, ListConfig)) + + with flag_override(target, ["readonly", "no_deepcopy_set_nodes"], [False, True]): + target.merge_with( + *configs[1:], + extend_lists=extend_lists, + remove_duplicates=remove_duplicates, + ) + turned_readonly = target._get_flag("readonly") is True + + if turned_readonly: + OmegaConf.set_readonly(target, True) + + return target + + @staticmethod + def register_resolver(name: str, resolver: Resolver) -> None: + warnings.warn( + dedent( + """\ + register_resolver() is deprecated. + See https://github.com/omry/omegaconf/issues/426 for migration instructions. + """ + ), + stacklevel=2, + ) + return OmegaConf.legacy_register_resolver(name, resolver) + + # This function will eventually be deprecated and removed. + @staticmethod + def legacy_register_resolver(name: str, resolver: Resolver) -> None: + assert callable(resolver), "resolver must be callable" + # noinspection PyProtectedMember + assert name not in BaseContainer._resolvers, f"resolver '{name}' is already registered" + + def resolver_wrapper( + config: BaseContainer, + parent: BaseContainer, + node: Node, + args: Tuple[Any, ...], + args_str: Tuple[str, ...], + ) -> Any: + cache = OmegaConf.get_cache(config)[name] + # "Un-escape " spaces and commas. + args_unesc = [x.replace(r"\ ", " ").replace(r"\,", ",") for x in args_str] + + # Nested interpolations behave in a potentially surprising way with + # legacy resolvers (they remain as strings, e.g., "${foo}"). If any + # input looks like an interpolation we thus raise an exception. + try: + bad_arg = next(i for i in args_unesc if "${" in i) + except StopIteration: + pass + else: + raise ValueError( + f"Resolver '{name}' was called with argument '{bad_arg}' that appears " + f"to be an interpolation. Nested interpolations are not supported for " + f"resolvers registered with `[legacy_]register_resolver()`, please use " + f"`register_new_resolver()` instead (see " + f"https://github.com/omry/omegaconf/issues/426 for migration instructions)." + ) + key = args_str + val = cache[key] if key in cache else resolver(*args_unesc) + cache[key] = val + return val + + # noinspection PyProtectedMember + BaseContainer._resolvers[name] = resolver_wrapper + + @staticmethod + def register_new_resolver( + name: str, + resolver: Resolver, + *, + replace: bool = False, + use_cache: bool = False, + ) -> None: + """ + Register a resolver. + + :param name: Name of the resolver. + :param resolver: Callable whose arguments are provided in the interpolation, + e.g., with ${foo:x,0,${y.z}} these arguments are respectively "x" (str), + 0 (int) and the value of ``y.z``. + :param replace: If set to ``False`` (default), then a ``ValueError`` is raised if + an existing resolver has already been registered with the same name. + If set to ``True``, then the new resolver replaces the previous one. + NOTE: The cache on existing config objects is not affected, use + ``OmegaConf.clear_cache(cfg)`` to clear it. + :param use_cache: Whether the resolver's outputs should be cached. The cache is + based only on the string literals representing the resolver arguments, e.g., + ${foo:${bar}} will always return the same value regardless of the value of + ``bar`` if the cache is enabled for ``foo``. + """ + if not callable(resolver): + raise TypeError("resolver must be callable") + if not name: + raise ValueError("cannot use an empty resolver name") + + if not replace and OmegaConf.has_resolver(name): + raise ValueError(f"resolver '{name}' is already registered") + + try: + sig: Optional[inspect.Signature] = inspect.signature(resolver) + except ValueError: + sig = None + + def _should_pass(special: str) -> bool: + ret = sig is not None and special in sig.parameters + if ret and use_cache: + raise ValueError(f"use_cache=True is incompatible with functions that receive the {special}") + return ret + + pass_parent = _should_pass("_parent_") + pass_node = _should_pass("_node_") + pass_root = _should_pass("_root_") + + def resolver_wrapper( + config: BaseContainer, + parent: Container, + node: Node, + args: Tuple[Any, ...], + args_str: Tuple[str, ...], + ) -> Any: + if use_cache: + cache = OmegaConf.get_cache(config)[name] + try: + return cache[args_str] + except KeyError: + pass + + # Call resolver. + kwargs: Dict[str, Node] = {} + if pass_parent: + kwargs["_parent_"] = parent + if pass_node: + kwargs["_node_"] = node + if pass_root: + kwargs["_root_"] = config + + ret = resolver(*args, **kwargs) + + if use_cache: + cache[args_str] = ret + return ret + + # noinspection PyProtectedMember + BaseContainer._resolvers[name] = resolver_wrapper + + @classmethod + def has_resolver(cls, name: str) -> bool: + return cls._get_resolver(name) is not None + + # noinspection PyProtectedMember + @staticmethod + def clear_resolvers() -> None: + """ + Clear(remove) all OmegaConf resolvers, then re-register OmegaConf's default resolvers. + """ + BaseContainer._resolvers = {} + register_default_resolvers() + + @classmethod + def clear_resolver(cls, name: str) -> bool: + """ + Clear(remove) any resolver only if it exists. + + Returns a bool: True if resolver is removed and False if not removed. + + .. warning: + This method can remove deafult resolvers as well. + + :param name: Name of the resolver. + :return: A bool (``True`` if resolver is removed, ``False`` if not found before removing). + """ + if cls.has_resolver(name): + BaseContainer._resolvers.pop(name) + return True + else: + # return False if resolver does not exist + return False + + @staticmethod + def get_cache(conf: BaseContainer) -> Dict[str, Any]: + return conf._metadata.resolver_cache + + @staticmethod + def set_cache(conf: BaseContainer, cache: Dict[str, Any]) -> None: + conf._metadata.resolver_cache = copy.deepcopy(cache) + + @staticmethod + def clear_cache(conf: BaseContainer) -> None: + OmegaConf.set_cache(conf, defaultdict(dict, {})) + + @staticmethod + def copy_cache(from_config: BaseContainer, to_config: BaseContainer) -> None: + OmegaConf.set_cache(to_config, OmegaConf.get_cache(from_config)) + + @staticmethod + def set_readonly(conf: Node, value: Optional[bool]) -> None: + # noinspection PyProtectedMember + conf._set_flag("readonly", value) + + @staticmethod + def is_readonly(conf: Node) -> Optional[bool]: + # noinspection PyProtectedMember + return conf._get_flag("readonly") + + @staticmethod + def set_struct(conf: Container, value: Optional[bool]) -> None: + # noinspection PyProtectedMember + conf._set_flag("struct", value) + + @staticmethod + def is_struct(conf: Container) -> Optional[bool]: + # noinspection PyProtectedMember + return conf._get_flag("struct") + + @staticmethod + def masked_copy(conf: DictConfig, keys: Union[str, List[str]]) -> DictConfig: + """ + Create a masked copy of of this config that contains a subset of the keys + + :param conf: DictConfig object + :param keys: keys to preserve in the copy + :return: The masked ``DictConfig`` object. + """ + from .dictconfig import DictConfig + + if not isinstance(conf, DictConfig): + raise ValueError("masked_copy is only supported for DictConfig") + + if isinstance(keys, str): + keys = [keys] + content = {key: value for key, value in conf.items_ex(resolve=False, keys=keys)} + return DictConfig(content=content) + + @staticmethod + def to_container( + cfg: Any, + *, + resolve: bool = False, + throw_on_missing: bool = False, + enum_to_str: bool = False, + structured_config_mode: SCMode = SCMode.DICT, + ) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: + """ + Recursively converts an OmegaConf config to a primitive container (dict or list). + + :param cfg: the config to convert + :param resolve: True to resolve all values + :param throw_on_missing: When True, raise MissingMandatoryValue if any missing values are present. + When False (the default), replace missing values with the string "???" in the output container. + :param enum_to_str: True to convert Enum keys and values to strings + :param structured_config_mode: Specify how Structured Configs (DictConfigs backed by a dataclass) are handled. + - By default (``structured_config_mode=SCMode.DICT``) structured configs are converted to plain dicts. + - If ``structured_config_mode=SCMode.DICT_CONFIG``, structured config nodes will remain as DictConfig. + - If ``structured_config_mode=SCMode.INSTANTIATE``, this function will instantiate structured configs + (DictConfigs backed by a dataclass), by creating an instance of the underlying dataclass. + + See also OmegaConf.to_object. + :return: A dict or a list representing this config as a primitive container. + """ + if not OmegaConf.is_config(cfg): + raise ValueError(f"Input cfg is not an OmegaConf config object ({type_str(type(cfg))})") + + return BaseContainer._to_content( + cfg, + resolve=resolve, + throw_on_missing=throw_on_missing, + enum_to_str=enum_to_str, + structured_config_mode=structured_config_mode, + ) + + @staticmethod + def to_object(cfg: Any) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: + """ + Recursively converts an OmegaConf config to a primitive container (dict or list). + Any DictConfig objects backed by dataclasses or attrs classes are instantiated + as instances of those backing classes. + + This is an alias for OmegaConf.to_container(..., resolve=True, throw_on_missing=True, + structured_config_mode=SCMode.INSTANTIATE) + + :param cfg: the config to convert + :return: A dict or a list or dataclass representing this config. + """ + return OmegaConf.to_container( + cfg=cfg, + resolve=True, + throw_on_missing=True, + enum_to_str=False, + structured_config_mode=SCMode.INSTANTIATE, + ) + + @staticmethod + def is_missing(cfg: Any, key: DictKeyType) -> bool: + assert isinstance(cfg, Container) + try: + node = cfg._get_child(key) + if node is None: + return False + assert isinstance(node, Node) + return node._is_missing() + except (UnsupportedInterpolationType, KeyError, AttributeError): + return False + + @staticmethod + def is_interpolation(node: Any, key: Optional[Union[int, str]] = None) -> bool: + if key is not None: + assert isinstance(node, Container) + target = node._get_child(key) + else: + target = node + if target is not None: + assert isinstance(target, Node) + return target._is_interpolation() + return False + + @staticmethod + def is_list(obj: Any) -> bool: + from . import ListConfig + + return isinstance(obj, ListConfig) + + @staticmethod + def is_dict(obj: Any) -> bool: + from . import DictConfig + + return isinstance(obj, DictConfig) + + @staticmethod + def is_config(obj: Any) -> bool: + from . import Container + + return isinstance(obj, Container) + + @staticmethod + def get_type(obj: Any, key: Optional[str] = None) -> Optional[Type[Any]]: + if key is not None: + c = obj._get_child(key) + else: + c = obj + return OmegaConf._get_obj_type(c) + + @staticmethod + def select( + cfg: Container, + key: str, + *, + default: Any = _DEFAULT_MARKER_, + throw_on_resolution_failure: bool = True, + throw_on_missing: bool = False, + ) -> Any: + """ + :param cfg: Config node to select from + :param key: Key to select + :param default: Default value to return if key is not found + :param throw_on_resolution_failure: Raise an exception if an interpolation + resolution error occurs, otherwise return None + :param throw_on_missing: Raise an exception if an attempt to select a missing key (with the value '???') + is made, otherwise return None + :return: selected value or None if not found. + """ + from ._impl import select_value + + try: + return select_value( + cfg=cfg, + key=key, + default=default, + throw_on_resolution_failure=throw_on_resolution_failure, + throw_on_missing=throw_on_missing, + ) + except Exception as e: + format_and_raise(node=cfg, key=key, value=None, cause=e, msg=str(e)) + + @staticmethod + def update( + cfg: Container, + key: str, + value: Any = None, + *, + merge: bool = True, + force_add: bool = False, + ) -> None: + """ + Updates a dot separated key sequence to a value + + :param cfg: input config to update + :param key: key to update (can be a dot separated path) + :param value: value to set, if value if a list or a dict it will be merged or set + depending on merge_config_values + :param merge: If value is a dict or a list, True (default) to merge + into the destination, False to replace the destination. + :param force_add: insert the entire path regardless of Struct flag or Structured Config nodes. + """ + + split = split_key(key) + root = cfg + for i in range(len(split) - 1): + k = split[i] + # if next_root is a primitive (string, int etc) replace it with an empty map + next_root, key_ = _select_one(root, k, throw_on_missing=False) + if not isinstance(next_root, Container): + if force_add: + with flag_override(root, "struct", False): + root[key_] = {} + else: + root[key_] = {} + root = root[key_] + + last = split[-1] + + assert isinstance(root, Container), f"Unexpected type for root: {type(root).__name__}" + + last_key: Union[str, int] = last + if isinstance(root, ListConfig): + last_key = int(last) + + ctx = flag_override(root, "struct", False) if force_add else nullcontext() + with ctx: + if merge and (OmegaConf.is_config(value) or is_primitive_container(value)): + assert isinstance(root, BaseContainer) + node = root._get_child(last_key) + if OmegaConf.is_config(node): + assert isinstance(node, BaseContainer) + node.merge_with(value) + return + + if OmegaConf.is_dict(root): + assert isinstance(last_key, str) + root.__setattr__(last_key, value) + elif OmegaConf.is_list(root): + assert isinstance(last_key, int) + root.__setitem__(last_key, value) + else: + assert False + + @staticmethod + def to_yaml(cfg: Any, *, resolve: bool = False, sort_keys: bool = False) -> str: + """ + returns a yaml dump of this config object. + + :param cfg: Config object, Structured Config type or instance + :param resolve: if True, will return a string with the interpolations resolved, otherwise + interpolations are preserved + :param sort_keys: If True, will print dict keys in sorted order. default False. + :return: A string containing the yaml representation. + """ + cfg = _ensure_container(cfg) + container = OmegaConf.to_container(cfg, resolve=resolve, enum_to_str=True) + return yaml.dump( # type: ignore + container, + default_flow_style=False, + allow_unicode=True, + sort_keys=sort_keys, + Dumper=get_omega_conf_dumper(), + ) + + @staticmethod + def resolve(cfg: Container) -> None: + """ + Resolves all interpolations in the given config object in-place. + + :param cfg: An OmegaConf container (DictConfig, ListConfig) + Raises a ValueError if the input object is not an OmegaConf container. + """ + import omegaconf._impl + + if not OmegaConf.is_config(cfg): + # Since this function is mutating the input object in-place, it doesn't make sense to + # auto-convert the input object to an OmegaConf container + raise ValueError(f"Invalid config type ({type(cfg).__name__}), expected an OmegaConf Container") + omegaconf._impl._resolve(cfg) + + @staticmethod + def missing_keys(cfg: Any) -> Set[str]: + """ + Returns a set of missing keys in a dotlist style. + + :param cfg: An ``OmegaConf.Container``, + or a convertible object via ``OmegaConf.create`` (dict, list, ...). + :return: set of strings of the missing keys. + :raises ValueError: On input not representing a config. + """ + cfg = _ensure_container(cfg) + missings: Set[str] = set() + + def gather(_cfg: Container) -> None: + itr: Iterable[Any] + if isinstance(_cfg, ListConfig): + itr = range(len(_cfg)) + else: + itr = _cfg + + for key in itr: + if OmegaConf.is_missing(_cfg, key): + missings.add(_cfg._get_full_key(key)) + elif OmegaConf.is_config(_cfg[key]): + gather(_cfg[key]) + + gather(cfg) + return missings + + # === private === # + + @staticmethod + def _create_impl( # noqa F811 + obj: Any = _DEFAULT_MARKER_, + parent: Optional[BaseContainer] = None, + flags: Optional[Dict[str, bool]] = None, + ) -> Union[DictConfig, ListConfig]: + try: + from ._utils import get_yaml_loader + from .dictconfig import DictConfig + from .listconfig import ListConfig + + if obj is _DEFAULT_MARKER_: + obj = {} + if isinstance(obj, str): + obj = yaml.load(obj, Loader=get_yaml_loader()) + if obj is None: + return OmegaConf.create({}, parent=parent, flags=flags) + elif isinstance(obj, str): + return OmegaConf.create({obj: None}, parent=parent, flags=flags) + else: + assert isinstance(obj, (list, dict)) + return OmegaConf.create(obj, parent=parent, flags=flags) + + else: + if ( + is_primitive_dict(obj) + or OmegaConf.is_dict(obj) + or is_structured_config(obj) + or obj is None + ): + if isinstance(obj, DictConfig): + return DictConfig( + content=obj, + parent=parent, + ref_type=obj._metadata.ref_type, + is_optional=obj._metadata.optional, + key_type=obj._metadata.key_type, + element_type=obj._metadata.element_type, + flags=flags, + ) + else: + obj_type = OmegaConf.get_type(obj) + key_type, element_type = get_dict_key_value_types(obj_type) + return DictConfig( + content=obj, + parent=parent, + key_type=key_type, + element_type=element_type, + flags=flags, + ) + elif is_primitive_list(obj) or OmegaConf.is_list(obj): + if isinstance(obj, ListConfig): + return ListConfig( + content=obj, + parent=parent, + element_type=obj._metadata.element_type, + ref_type=obj._metadata.ref_type, + is_optional=obj._metadata.optional, + flags=flags, + ) + else: + obj_type = OmegaConf.get_type(obj) + element_type = get_list_element_type(obj_type) + return ListConfig( + content=obj, + parent=parent, + element_type=element_type, + ref_type=Any, + is_optional=True, + flags=flags, + ) + else: + if isinstance(obj, type): + raise ValidationError( + f"Input class '{obj.__name__}' is not a structured config. " + "did you forget to decorate it as a dataclass?" + ) + else: + raise ValidationError(f"Object of unsupported type: '{type(obj).__name__}'") + except OmegaConfBaseException as e: + format_and_raise(node=None, key=None, value=None, msg=str(e), cause=e) + assert False + + @staticmethod + def _get_obj_type(c: Any) -> Optional[Type[Any]]: + if is_structured_config(c): + return get_type_of(c) + elif c is None: + return None + elif isinstance(c, DictConfig): + if c._is_none(): + return None + elif c._is_missing(): + return None + else: + if is_structured_config(c._metadata.object_type): + return c._metadata.object_type + else: + return dict + elif isinstance(c, ListConfig): + return list + elif isinstance(c, ValueNode): + return type(c._value()) + elif isinstance(c, UnionNode): + return type(_get_value(c)) + elif isinstance(c, dict): + return dict + elif isinstance(c, (list, tuple)): + return list + else: + return get_type_of(c) + + @staticmethod + def _get_resolver( + name: str, + ) -> Optional[Callable[[Container, Container, Node, Tuple[Any, ...], Tuple[str, ...]], Any,]]: + # noinspection PyProtectedMember + return BaseContainer._resolvers[name] if name in BaseContainer._resolvers else None + + +# register all default resolvers +register_default_resolvers() + + +@contextmanager +def flag_override( + config: Node, + names: Union[List[str], str], + values: Union[List[Optional[bool]], Optional[bool]], +) -> Generator[Node, None, None]: + if isinstance(names, str): + names = [names] + if values is None or isinstance(values, bool): + values = [values] + + prev_states = [config._get_node_flag(name) for name in names] + + try: + config._set_flag(names, values) + yield config + finally: + config._set_flag(names, prev_states) + + +@contextmanager +def read_write(config: Node) -> Generator[Node, None, None]: + prev_state = config._get_node_flag("readonly") + try: + OmegaConf.set_readonly(config, False) + yield config + finally: + OmegaConf.set_readonly(config, prev_state) + + +@contextmanager +def open_dict(config: Container) -> Generator[Container, None, None]: + prev_state = config._get_node_flag("struct") + try: + OmegaConf.set_struct(config, False) + yield config + finally: + OmegaConf.set_struct(config, prev_state) + + +# === private === # + + +def _node_wrap( + parent: Optional[Box], + is_optional: bool, + value: Any, + key: Any, + ref_type: Any = Any, +) -> Node: + node: Node + if is_dict_annotation(ref_type) or (is_primitive_dict(value) and ref_type is Any): + key_type, element_type = get_dict_key_value_types(ref_type) + node = DictConfig( + content=value, + key=key, + parent=parent, + ref_type=ref_type, + is_optional=is_optional, + key_type=key_type, + element_type=element_type, + ) + elif (is_list_annotation(ref_type) or is_tuple_annotation(ref_type)) or ( + type(value) in (list, tuple) and ref_type is Any + ): + element_type = get_list_element_type(ref_type) + node = ListConfig( + content=value, + key=key, + parent=parent, + is_optional=is_optional, + element_type=element_type, + ref_type=ref_type, + ) + elif is_structured_config(ref_type) or is_structured_config(value): + key_type, element_type = get_dict_key_value_types(value) + node = DictConfig( + ref_type=ref_type, + is_optional=is_optional, + content=value, + key=key, + parent=parent, + key_type=key_type, + element_type=element_type, + ) + elif is_union_annotation(ref_type): + node = UnionNode( + content=value, + ref_type=ref_type, + is_optional=is_optional, + key=key, + parent=parent, + ) + elif ref_type == Any or ref_type is None: + node = AnyNode(value=value, key=key, parent=parent) + elif isinstance(ref_type, type) and issubclass(ref_type, Enum): + node = EnumNode( + enum_type=ref_type, + value=value, + key=key, + parent=parent, + is_optional=is_optional, + ) + elif ref_type == int: + node = IntegerNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == float: + node = FloatNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == bool: + node = BooleanNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == str: + node = StringNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == bytes: + node = BytesNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == pathlib.Path: + node = PathNode(value=value, key=key, parent=parent, is_optional=is_optional) + else: + if parent is not None and parent._get_flag("allow_objects") is True: + if type(value) in (list, tuple): + node = ListConfig( + content=value, + key=key, + parent=parent, + ref_type=ref_type, + is_optional=is_optional, + ) + elif is_primitive_dict(value): + node = DictConfig( + content=value, + key=key, + parent=parent, + ref_type=ref_type, + is_optional=is_optional, + ) + else: + node = AnyNode(value=value, key=key, parent=parent) + else: + raise ValidationError(f"Unexpected type annotation: {type_str(ref_type)}") + return node + + +def _maybe_wrap( + ref_type: Any, + key: Any, + value: Any, + is_optional: bool, + parent: Optional[BaseContainer], +) -> Node: + # if already a node, update key and parent and return as is. + # NOTE: that this mutate the input node! + if isinstance(value, Node): + value._set_key(key) + value._set_parent(parent) + return value + else: + return _node_wrap( + ref_type=ref_type, + parent=parent, + is_optional=is_optional, + value=value, + key=key, + ) + + +def _select_one( + c: Container, key: str, throw_on_missing: bool, throw_on_type_error: bool = True +) -> Tuple[Optional[Node], Union[str, int]]: + from .dictconfig import DictConfig + from .listconfig import ListConfig + + ret_key: Union[str, int] = key + assert isinstance(c, Container), f"Unexpected type: {c}" + if c._is_none(): + return None, ret_key + + if isinstance(c, DictConfig): + assert isinstance(ret_key, str) + val = c._get_child(ret_key, validate_access=False) + elif isinstance(c, ListConfig): + assert isinstance(ret_key, str) + if not is_int(ret_key): + if throw_on_type_error: + raise TypeError(f"Index '{ret_key}' ({type(ret_key).__name__}) is not an int") + else: + val = None + else: + ret_key = int(ret_key) + if ret_key < 0 or ret_key + 1 > len(c): + val = None + else: + val = c._get_child(ret_key) + else: + assert False + + if val is not None: + assert isinstance(val, Node) + if val._is_missing(): + if throw_on_missing: + raise MissingMandatoryValue(f"Missing mandatory value: {c._get_full_key(ret_key)}") + else: + return val, ret_key + + assert val is None or isinstance(val, Node) + return val, ret_key diff --git a/omegaconf/py.typed b/omegaconf/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/omegaconf/resolvers/__init__.py b/omegaconf/resolvers/__init__.py new file mode 100644 index 000000000..1b0bb7116 --- /dev/null +++ b/omegaconf/resolvers/__init__.py @@ -0,0 +1,5 @@ +from omegaconf.resolvers import oc + +__all__ = [ + "oc", +] diff --git a/omegaconf/resolvers/oc/__init__.py b/omegaconf/resolvers/oc/__init__.py new file mode 100644 index 000000000..a8e0e9e66 --- /dev/null +++ b/omegaconf/resolvers/oc/__init__.py @@ -0,0 +1,109 @@ +import os +import string +import warnings +from typing import Any, Optional + +from omegaconf import Container, Node +from omegaconf._utils import _DEFAULT_MARKER_, _get_value +from omegaconf.basecontainer import BaseContainer +from omegaconf.errors import ConfigKeyError +from omegaconf.grammar_parser import parse +from omegaconf.resolvers.oc import dict + + +def create(obj: Any, _parent_: Container) -> Any: + """Create a config object from `obj`, similar to `OmegaConf.create`""" + from omegaconf import OmegaConf + + assert isinstance(_parent_, BaseContainer) + return OmegaConf.create(obj, parent=_parent_) + + +def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]: + """ + :param key: Environment variable key + :param default: Optional default value to use in case the key environment variable is not set. + If default is not a string, it is converted with str(default). + None default is returned as is. + :return: The environment variable 'key'. If the environment variable is not set and a default is + provided, the default is used. If used, the default is converted to a string with str(default). + If the default is None, None is returned (without a string conversion). + """ + try: + return os.environ[key] + except KeyError: + if default is not _DEFAULT_MARKER_: + return str(default) if default is not None else None + else: + raise KeyError(f"Environment variable '{key}' not found") + + +def decode(expr: Optional[str], _parent_: Container, _node_: Node) -> Any: + """ + Parse and evaluate `expr` according to the `singleElement` rule of the grammar. + + If `expr` is `None`, then return `None`. + """ + if expr is None: + return None + + if not isinstance(expr, str): + raise TypeError( + f"`oc.decode` can only take strings or None as input, " + f"but `{expr}` is of type {type(expr).__name__}" + ) + + parse_tree = parse(expr, parser_rule="singleElement", lexer_mode="VALUE_MODE") + val = _parent_.resolve_parse_tree(parse_tree, node=_node_) + return _get_value(val) + + +def deprecated( + key: str, + message: str = "'$OLD_KEY' is deprecated. Change your code and config to use '$NEW_KEY'", + *, + _parent_: Container, + _node_: Node, +) -> Any: + from omegaconf._impl import select_node + + if not isinstance(key, str): + raise TypeError(f"oc.deprecated: interpolation key type is not a string ({type(key).__name__})") + + if not isinstance(message, str): + raise TypeError( + f"oc.deprecated: interpolation message type is not a string ({type(message).__name__})" + ) + + full_key = _node_._get_full_key(key=None) + target_node = select_node(_parent_, key, absolute_key=True) + if target_node is None: + raise ConfigKeyError(f"In oc.deprecated resolver at '{full_key}': Key not found: '{key}'") + new_key = target_node._get_full_key(key=None) + msg = string.Template(message).safe_substitute( + OLD_KEY=full_key, + NEW_KEY=new_key, + ) + warnings.warn(category=UserWarning, message=msg) + return target_node + + +def select( + key: str, + default: Any = _DEFAULT_MARKER_, + *, + _parent_: Container, +) -> Any: + from omegaconf._impl import select_value + + return select_value(cfg=_parent_, key=key, absolute_key=True, default=default) + + +__all__ = [ + "create", + "decode", + "deprecated", + "dict", + "env", + "select", +] diff --git a/omegaconf/resolvers/oc/dict.py b/omegaconf/resolvers/oc/dict.py new file mode 100644 index 000000000..3bcb06b8f --- /dev/null +++ b/omegaconf/resolvers/oc/dict.py @@ -0,0 +1,78 @@ +from typing import Any, List + +from omegaconf import AnyNode, Container, DictConfig, ListConfig +from omegaconf._utils import Marker +from omegaconf.basecontainer import BaseContainer +from omegaconf.errors import ConfigKeyError + +_DEFAULT_SELECT_MARKER_: Any = Marker("_DEFAULT_SELECT_MARKER_") + + +def keys( + key: str, + _parent_: Container, +) -> ListConfig: + from omegaconf import OmegaConf + + assert isinstance(_parent_, BaseContainer) + + in_dict = _get_and_validate_dict_input(key, parent=_parent_, resolver_name="oc.dict.keys") + + ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_) + assert isinstance(ret, ListConfig) + return ret + + +def values(key: str, _root_: BaseContainer, _parent_: Container) -> ListConfig: + assert isinstance(_parent_, BaseContainer) + in_dict = _get_and_validate_dict_input(key, parent=_parent_, resolver_name="oc.dict.values") + + content = in_dict._content + assert isinstance(content, dict) + + ret = ListConfig([]) + if key.startswith("."): + key = f".{key}" # extra dot to compensate for extra level of nesting within ret ListConfig + for k in content: + ref_node = AnyNode(f"${{{key}.{k!s}}}") + ret.append(ref_node) + + # Finalize result by setting proper type and parent. + element_type: Any = in_dict._metadata.element_type + ret._metadata.element_type = element_type + ret._metadata.ref_type = List[element_type] + ret._set_parent(_parent_) + + return ret + + +def _get_and_validate_dict_input( + key: str, + parent: BaseContainer, + resolver_name: str, +) -> DictConfig: + from omegaconf._impl import select_value + + if not isinstance(key, str): + raise TypeError( + f"`{resolver_name}` requires a string as input, but obtained `{key}` " + f"of type: {type(key).__name__}" + ) + + in_dict = select_value( + parent, + key, + throw_on_missing=True, + absolute_key=True, + default=_DEFAULT_SELECT_MARKER_, + ) + + if in_dict is _DEFAULT_SELECT_MARKER_: + raise ConfigKeyError(f"Key not found: '{key}'") + + if not isinstance(in_dict, DictConfig): + raise TypeError( + f"`{resolver_name}` cannot be applied to objects of type: " f"{type(in_dict).__name__}" + ) + + return in_dict diff --git a/omegaconf/version.py b/omegaconf/version.py new file mode 100644 index 000000000..f73c6b765 --- /dev/null +++ b/omegaconf/version.py @@ -0,0 +1,13 @@ +import sys # pragma: no cover + +__version__ = "2.4.0.dev0" + +msg = """OmegaConf 2.0 and above is compatible with Python 3.6 and newer. +You have the following options: +1. Upgrade to Python 3.6 or newer. + This is highly recommended. new features will not be added to OmegaConf 1.4. +2. Continue using OmegaConf 1.4: + You can pip install 'OmegaConf<1.5' to do that. +""" +if sys.version_info < (3, 6): + raise ImportError(msg) # pragma: no cover From 6b54600a7f86a4303b622eea91b80793976faf00 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:46:32 +0200 Subject: [PATCH 10/37] add migrated inventory to test the feature --- .../classes/cluster/common.yml | 3 + .../classes/cluster/minikube.yml | 19 ++++++ .../inventory_omegaconf/classes/common.yml | 18 ++++++ .../classes/component/busybox.yml | 19 ++++++ .../classes/component/elasticsearch.yml | 46 ++++++++++++++ .../classes/component/labels.yml | 16 +++++ .../classes/component/mysql.yml | 61 +++++++++++++++++++ .../classes/component/namespace.yml | 9 +++ .../classes/component/nginx-common.yml | 12 ++++ .../classes/component/nginx-helm.yml | 24 ++++++++ .../classes/component/nginx-jsonnet.yml | 12 ++++ .../classes/component/nginx-kadet.yml | 12 ++++ .../inventory_omegaconf/targets/all-glob.yml | 60 ++++++++++++++++++ .../inventory_omegaconf/targets/busybox.yml | 6 ++ .../inventory_omegaconf/targets/labels.yml | 6 ++ .../targets/minikube-es.yml | 11 ++++ .../targets/minikube-mysql.yml | 9 +++ .../targets/minikube-nginx-helm.yml | 9 +++ .../targets/minikube-nginx-jsonnet.yml | 10 +++ .../targets/minikube-nginx-kadet.yml | 11 ++++ .../inventory_omegaconf/targets/removal.yml | 14 +++++ 21 files changed, 387 insertions(+) create mode 100644 examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/common.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/labels.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml create mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/all-glob.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/busybox.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/labels.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml create mode 100644 examples/kubernetes/inventory_omegaconf/targets/removal.yml diff --git a/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml b/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml new file mode 100644 index 000000000..36ed23592 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml @@ -0,0 +1,3 @@ +parameters: + kubectl: + insecure_skip_tls_verify: false diff --git a/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml b/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml new file mode 100644 index 000000000..b8ee0ebc5 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml @@ -0,0 +1,19 @@ +# class for all targets deployed on minikube +# +classes: + - cluster.common +parameters: + minikube: + memory: 4096 + cpus: 4 + version: v0.31.0 + + cluster: + type: minikube + id: minikube + name: minikube + user: minikube + vault: + address: https://localhost:8200 + mysql: + hostname: localhost diff --git a/examples/kubernetes/inventory_omegaconf/classes/common.yml b/examples/kubernetes/inventory_omegaconf/classes/common.yml new file mode 100644 index 000000000..06e36f4d0 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/common.yml @@ -0,0 +1,18 @@ +parameters: + namespace: ${parameters.target_name} + target_name: ${parameters._reclass_.name.short} + + kapitan: + vars: + target: ${parameters.target_name} + namespace: ${parameters.target_name} + managed_by: kapitan + secrets: + gpg: + recipients: + - name: example@kapitan.dev + fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C + gkms: + key: 'projects//locations//keyRings//cryptoKeys/' + awskms: + key: 'alias/nameOfKey' diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml b/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml new file mode 100644 index 000000000..f664cde51 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml @@ -0,0 +1,19 @@ +parameters: + kapitan: + vars: + target: ${parameters.target_name} + namespace: ${parameters.target_name} + compile: + - output_path: ./copy + input_type: copy + input_paths: + - components/busybox/pod.yml + - input_type: copy + input_paths: + - copy_target + output_path: ./copy + # test copying over root output_path + - input_type: copy + input_paths: + - copy_target + output_path: . diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml b/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml new file mode 100644 index 000000000..4d88ad628 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml @@ -0,0 +1,46 @@ +parameters: + elasticsearch: + image: "quay.io/pires/docker-elasticsearch-kubernetes:5.5.0" + java_opts: "-Xms512m -Xmx512m" + replicas: 1 + masters: 1 + roles: + master: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + data: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + client: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + ingest: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + + kapitan: + vars: + target: ${parameters.target_name} + namespace: ${parameters.target_name} + compile: + - output_path: manifests + input_type: jsonnet + input_paths: + - components/elasticsearch/main.jsonnet + output_type: yml + - output_path: scripts + input_type: jinja2 + input_paths: + - scripts + - output_path: . + input_type: jinja2 + input_paths: + - docs/elasticsearch/README.md diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml b/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml new file mode 100644 index 000000000..398be2969 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml @@ -0,0 +1,16 @@ +parameters: + postgres: + name: db + component: db + instance: postgres + version: "11.0" + + kapitan: + vars: + target: ${parameters.target_name} + namespace: ${parameters.target_name} + compile: + - output_path: ./labels + input_type: jinja2 + input_paths: + - components/labels/service.yml diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml b/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml new file mode 100644 index 000000000..b0f24a2ab --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml @@ -0,0 +1,61 @@ +parameters: + kapitan: + compile: + - output_path: manifests + input_type: jsonnet + prune: true + input_paths: + - components/mysql/main.jsonnet + output_type: yml + - output_path: scripts + input_type: jinja2 + input_paths: + - scripts + - output_path: . + output_type: yml + input_type: jinja2 + input_paths: + - docs/mysql/README.md + validate: + - type: kubernetes # mkdocs (1)! + output_paths: # mkdocs (2)! + - manifests/mysql_secret.yml + kind: secret # temporarily replaced with 'deployment' during test # mkdocs (3)! + version: 1.14.0 # optional, defaults to 1.14.0 # mkdocs (4)! + - type: kubernetes + output_paths: + - manifests/mysql_service_jsonnet.yml + - manifests/mysql_service_simple.yml + kind: service + version: 1.14.0 + # For vaultkv secrets it is important to declare auth in parameters + secrets: + vaultkv: + auth: token + vaulttransit: + auth: token + key: my-vault-key + mysql: + storage: 10G + storage_class: standard + image: mysql:latest + users: + root: + # If 'secrets/targets/${target_name}/mysql/password' doesn't exist, it will gen a random b64-encoded password + password: ?{gpg:targets/${parameters.target_name}/mysql/password||randomstr|base64} + # password: ?{gkms:targets/${target_name}/mysql/password||randomstr|base64} + # password: ?{awskms:targets/${target_name}/mysql/password||randomstr|base64} + + # Generates the sha256 checksum of the previously declared B64'ed password + # It's base64'ed again so that it can be used in kubernetes secrets + password_sha256: + ?{gpg:targets/${parameters.target_name}/mysql/password_sha256||reveal:targets/${parameters.target_name}/mysql/password|sha256|base64} + + password_subvar: ?{gpg:targets/${parameters.target_name}/mysql/subvars@var1.password} + password_sha256_subvar: ?{gpg:targets/${parameters.target_name}/mysql/subvars@var2.password_sha256} + + # This secret requires a running vault instance & vaultkv parameters either in inventory or environment + # password_vaultkv: ?{vaultkv:targets/${target_name}/mysql/vault_secret} + + # This secret requires a running vault instance & vaulttransit parameters either in inventory or environment + # password_transit: ?{transit:${target_name}/mysql/vault_secret} diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml b/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml new file mode 100644 index 000000000..ba82d74ac --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml @@ -0,0 +1,9 @@ +parameters: + namespace: ${parameters.target_name} + kapitan: + compile: + - output_path: pre-deploy + input_type: jsonnet + output_type: yml + input_paths: + - components/namespace/main.jsonnet diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml new file mode 100644 index 000000000..f4d62f1a3 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml @@ -0,0 +1,12 @@ +parameters: + nginx: + image: nginx:1:15.8 + + templates: #(1)! + - docs/nginx/README.md + - components/nginx-deploy.sh + kapitan: + compile: + - output_path: . #(2)! + input_type: jinja2 + input_paths: ${parameters.templates} #(3)! diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml new file mode 100644 index 000000000..18b944e3a --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml @@ -0,0 +1,24 @@ +parameters: + namespace: + nginx: + version: 4.4.0 + replicas: 2 + name: ${parameters.target_name} + namespace: ${parameters.target_name} + kapitan: + dependencies: + - type: helm + output_path: charts/nginx-ingress + source: https://kubernetes.github.io/ingress-nginx + chart_name: ingress-nginx + compile: + - output_path: . + input_type: helm + input_paths: + - charts/nginx-ingress + helm_values: + controller: + name: ${parameters.nginx.name} + helm_params: + name: ${parameters.nginx.name} + namespace: ${parameters.nginx.namespace} diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml new file mode 100644 index 000000000..773c33e68 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml @@ -0,0 +1,12 @@ +parameters: + nginx: + replicas: 1 + kapitan: + compile: + - output_path: manifests + output_type: yml + input_type: jsonnet + input_paths: + - components/nginx-jsonnet/main.jsonnet + labels: + type: jsonnet diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml new file mode 100644 index 000000000..ff9e2f919 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml @@ -0,0 +1,12 @@ +parameters: + nginx: + replicas: 2 + kapitan: + compile: + - output_path: manifests + input_type: kadet + output_type: yml + input_paths: + - components/nginx-kadet/ + labels: + type: kadet diff --git a/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml b/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml new file mode 100644 index 000000000..319c852b7 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml @@ -0,0 +1,60 @@ +classes: + - common + - component.namespace + - cluster.minikube +parameters: + target_name: all-glob + + elasticsearch: + image: "quay.io/pires/docker-elasticsearch-kubernetes:5.5.0" + java_opts: "-Xms512m -Xmx512m" + replicas: 1 + masters: 1 + roles: + ingest: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + master: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + data: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + client: + image: ${parameters.elasticsearch.image} + java_opts: ${parameters.elasticsearch.java_opts} + replicas: ${parameters.elasticsearch.replicas} + masters: ${parameters.elasticsearch.masters} + + mysql: + instance_name: glob_instance + image: mysql:latest + storage_class: standard + storage: 10G + users: + root: + password: ?{base64:targets/${parameters.target_name}/mysql/password||randomstr|base64} + password_sha256: + ?{base64:targets/${parameters.target_name}/mysql/password_sha256||reveal:targets/${parameters.target_name}/mysql/password|sha256|base64} + password_subvar: ?{base64:targets/${parameters.target_name}/mysql/subvars@var1.password} + password_sha256_subvar: ?{base64:targets/${parameters.target_name}/mysql/subvars@var2.password_sha256} + + nginx: + image: nginx:1:15.8 + replicas: 2 + + # compile all main.jsonnet in components/ + kapitan: + compile: + - output_path: manifests + input_type: jsonnet + input_paths: + - components/*/main.jsonnet + output_type: yml + diff --git a/examples/kubernetes/inventory_omegaconf/targets/busybox.yml b/examples/kubernetes/inventory_omegaconf/targets/busybox.yml new file mode 100644 index 000000000..16622e715 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/busybox.yml @@ -0,0 +1,6 @@ +classes: + - common + - component.namespace + - component.busybox +parameters: + target_name: busybox diff --git a/examples/kubernetes/inventory_omegaconf/targets/labels.yml b/examples/kubernetes/inventory_omegaconf/targets/labels.yml new file mode 100644 index 000000000..2215b5349 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/labels.yml @@ -0,0 +1,6 @@ +classes: + - common + - component.namespace + - component.labels +parameters: + target_name: labels diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml new file mode 100644 index 000000000..c33985517 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml @@ -0,0 +1,11 @@ +classes: + - common + - cluster.minikube + - component.namespace + - component.elasticsearch + - component.busybox +parameters: + target_name: minikube-es + + elasticsearch: + replicas: 2 diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml new file mode 100644 index 000000000..1d32880c6 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml @@ -0,0 +1,9 @@ +classes: + - common + - cluster.minikube + - component.namespace + - component.mysql +parameters: + mysql: + instance_name: example-mysql + replicas: 1 diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml new file mode 100644 index 000000000..f9f8162e8 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml @@ -0,0 +1,9 @@ +classes: + - common + - cluster.minikube + - component.nginx-helm + - component.nginx-common +parameters: +# These parameters are redundand because automatically set by `common` +## target_name: minikube-nginx-helm +## namespace: ${target_name} diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml new file mode 100644 index 000000000..439a62f28 --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml @@ -0,0 +1,10 @@ +classes: + - common + - cluster.minikube + - component.namespace + - component.nginx-jsonnet + - component.nginx-common +parameters: +# These parameters are redundand because automatically set by `common` +## target_name: minikube-nginx-jsonnet +## namespace: ${target_name} diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml new file mode 100644 index 000000000..09540fc5e --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml @@ -0,0 +1,11 @@ +classes: + - common + - cluster.minikube + - component.namespace + - component.nginx-kadet + - component.nginx-common +parameters: +# These parameters are redundand because automatically set by `common` +## target_name: minikube-nginx-kadet +## namespace: ${target_name} + diff --git a/examples/kubernetes/inventory_omegaconf/targets/removal.yml b/examples/kubernetes/inventory_omegaconf/targets/removal.yml new file mode 100644 index 000000000..c42ba1b2e --- /dev/null +++ b/examples/kubernetes/inventory_omegaconf/targets/removal.yml @@ -0,0 +1,14 @@ +classes: + - common +parameters: + kapitan: + compile: + - input_type: copy + input_paths: + - copy_target + output_path: . + # test removal of a file + - input_type: remove + input_paths: + - compiled/${parameters.kapitan.vars.target}/copy_target + output_path: . From 60c56ea690999c2c973c37ed81d6e987bd81334b Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:47:56 +0200 Subject: [PATCH 11/37] refactor: add error handling and correct merging flags --- kapitan/migrate_omegaconf.py | 2 -- kapitan/resources.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py index a7fa3f1e8..8942ffb26 100644 --- a/kapitan/migrate_omegaconf.py +++ b/kapitan/migrate_omegaconf.py @@ -74,7 +74,6 @@ def migrate_file(input_file: str) -> None: def migrate(inv_path: str = "", output_path: str = "") -> None: - targets_path = os.path.join(inv_path, "targets") classes_path = os.path.join(inv_path, "classes") @@ -91,7 +90,6 @@ def migrate(inv_path: str = "", output_path: str = "") -> None: # support running the file without kapitan if __name__ == "__main__": - if len(sys.argv) != 2: print("Usage: migrate_omegaconf.py INV_PATH") sys.exit(1) diff --git a/kapitan/resources.py b/kapitan/resources.py index 7c654e6ec..0a2499f99 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -459,15 +459,21 @@ def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], target_config_classes.extend(class_config.pop("classes", [])) elif not ignore_class_notfound: - raise InventoryError(f"Target: {target_name}: Class {class_name} not found.") + raise InventoryError(f"{target_name}: Class {class_name} not found.") else: continue # merge target with loaded classes - target_config = OmegaConf.merge(target_config, class_config) + if target_config.get("parameters"): + target_config = OmegaConf.merge(class_config, target_config, extend_lists=True) + else: + target_config = class_config if not target_config: - logger.warning(f"{target_name}: empty target") + raise InventoryError(f"{target_name}: empty target") + + if not target_config.get("parameters"): + raise InventoryError(f"{target_name}: target has no parameters") # append meta data _reclass_ (legacy) (refactoring TBD) target_config["parameters"]["_reclass_"] = { From b1f85e70a8e4b3b9bafeeba5b6905c0551045f15 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:49:29 +0200 Subject: [PATCH 12/37] deps: add temporarily deps ruamel and regex for migrating --- poetry.lock | 193 ++++++++++++++++++++++++++++++++++++++++++------- pyproject.toml | 3 +- 2 files changed, 167 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index f7473ee3f..96bf2936e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,17 +12,6 @@ files = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] -[[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -description = "ANTLR 4.9.3 runtime for Python 3.7" -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, -] - [[package]] name = "attrs" version = "22.2.0" @@ -828,22 +817,6 @@ portalocker = [ {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, ] -[[package]] -name = "omegaconf" -version = "2.3.0" -description = "A flexible configuration library" -category = "main" -optional = false -python-versions = ">=3.6" -files = [ - {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, - {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, -] - -[package.dependencies] -antlr4-python3-runtime = ">=4.9.0,<4.10.0" -PyYAML = ">=5.1.0" - [[package]] name = "packaging" version = "23.0" @@ -1231,6 +1204,104 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "regex" +version = "2023.5.5" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "regex-2023.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:48c9ec56579d4ba1c88f42302194b8ae2350265cb60c64b7b9a88dcb7fbde309"}, + {file = "regex-2023.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53e22e4460f0245b468ee645156a4f84d0fc35a12d9ba79bd7d79bdcd2f9629d"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b870b6f632fc74941cadc2a0f3064ed8409e6f8ee226cdfd2a85ae50473aa94"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:171c52e320fe29260da550d81c6b99f6f8402450dc7777ef5ced2e848f3b6f8f"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad5524c2aedaf9aa14ef1bc9327f8abd915699dea457d339bebbe2f0d218f86"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a0f874ee8c0bc820e649c900243c6d1e6dc435b81da1492046716f14f1a2a96"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e645c757183ee0e13f0bbe56508598e2d9cd42b8abc6c0599d53b0d0b8dd1479"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a4c5da39bca4f7979eefcbb36efea04471cd68db2d38fcbb4ee2c6d440699833"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5e3f4468b8c6fd2fd33c218bbd0a1559e6a6fcf185af8bb0cc43f3b5bfb7d636"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:59e4b729eae1a0919f9e4c0fc635fbcc9db59c74ad98d684f4877be3d2607dd6"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ba73a14e9c8f9ac409863543cde3290dba39098fc261f717dc337ea72d3ebad2"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bbd5dcb19603ab8d2781fac60114fb89aee8494f4505ae7ad141a3314abb1f9"}, + {file = "regex-2023.5.5-cp310-cp310-win32.whl", hash = "sha256:40005cbd383438aecf715a7b47fe1e3dcbc889a36461ed416bdec07e0ef1db66"}, + {file = "regex-2023.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:59597cd6315d3439ed4b074febe84a439c33928dd34396941b4d377692eca810"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f08276466fedb9e36e5193a96cb944928301152879ec20c2d723d1031cd4ddd"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cd46f30e758629c3ee91713529cfbe107ac50d27110fdcc326a42ce2acf4dafc"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2910502f718828cecc8beff004917dcf577fc5f8f5dd40ffb1ea7612124547b"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:445d6f4fc3bd9fc2bf0416164454f90acab8858cd5a041403d7a11e3356980e8"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18196c16a584619c7c1d843497c069955d7629ad4a3fdee240eb347f4a2c9dbe"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d430a23b661629661f1fe8395be2004006bc792bb9fc7c53911d661b69dd7e"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72a28979cc667e5f82ef433db009184e7ac277844eea0f7f4d254b789517941d"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f764e4dfafa288e2eba21231f455d209f4709436baeebb05bdecfb5d8ddc3d35"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23d86ad2121b3c4fc78c58f95e19173790e22ac05996df69b84e12da5816cb17"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:690a17db524ee6ac4a27efc5406530dd90e7a7a69d8360235323d0e5dafb8f5b"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:1ecf3dcff71f0c0fe3e555201cbe749fa66aae8d18f80d2cc4de8e66df37390a"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:811040d7f3dd9c55eb0d8b00b5dcb7fd9ae1761c454f444fd9f37fe5ec57143a"}, + {file = "regex-2023.5.5-cp311-cp311-win32.whl", hash = "sha256:c8c143a65ce3ca42e54d8e6fcaf465b6b672ed1c6c90022794a802fb93105d22"}, + {file = "regex-2023.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:586a011f77f8a2da4b888774174cd266e69e917a67ba072c7fc0e91878178a80"}, + {file = "regex-2023.5.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b6365703e8cf1644b82104cdd05270d1a9f043119a168d66c55684b1b557d008"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56c18f21ac98209da9c54ae3ebb3b6f6e772038681d6cb43b8d53da3b09ee81"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8b942d8b3ce765dbc3b1dad0a944712a89b5de290ce8f72681e22b3c55f3cc8"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:844671c9c1150fcdac46d43198364034b961bd520f2c4fdaabfc7c7d7138a2dd"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2ce65bdeaf0a386bb3b533a28de3994e8e13b464ac15e1e67e4603dd88787fa"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fee0016cc35a8a91e8cc9312ab26a6fe638d484131a7afa79e1ce6165328a135"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:18f05d14f14a812fe9723f13afafefe6b74ca042d99f8884e62dbd34dcccf3e2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:941b3f1b2392f0bcd6abf1bc7a322787d6db4e7457be6d1ffd3a693426a755f2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:921473a93bcea4d00295799ab929522fc650e85c6b9f27ae1e6bb32a790ea7d3"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:e2205a81f815b5bb17e46e74cc946c575b484e5f0acfcb805fb252d67e22938d"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:385992d5ecf1a93cb85adff2f73e0402dd9ac29b71b7006d342cc920816e6f32"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:890a09cb0a62198bff92eda98b2b507305dd3abf974778bae3287f98b48907d3"}, + {file = "regex-2023.5.5-cp36-cp36m-win32.whl", hash = "sha256:821a88b878b6589c5068f4cc2cfeb2c64e343a196bc9d7ac68ea8c2a776acd46"}, + {file = "regex-2023.5.5-cp36-cp36m-win_amd64.whl", hash = "sha256:7918a1b83dd70dc04ab5ed24c78ae833ae8ea228cef84e08597c408286edc926"}, + {file = "regex-2023.5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:338994d3d4ca4cf12f09822e025731a5bdd3a37aaa571fa52659e85ca793fb67"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a69cf0c00c4d4a929c6c7717fd918414cab0d6132a49a6d8fc3ded1988ed2ea"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f5e06df94fff8c4c85f98c6487f6636848e1dc85ce17ab7d1931df4a081f657"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8906669b03c63266b6a7693d1f487b02647beb12adea20f8840c1a087e2dfb5"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fda3e50abad8d0f48df621cf75adc73c63f7243cbe0e3b2171392b445401550"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ac2b7d341dc1bd102be849d6dd33b09701223a851105b2754339e390be0627a"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fb2b495dd94b02de8215625948132cc2ea360ae84fe6634cd19b6567709c8ae2"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aa7d032c1d84726aa9edeb6accf079b4caa87151ca9fabacef31fa028186c66d"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3d45864693351c15531f7e76f545ec35000d50848daa833cead96edae1665559"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21e90a288e6ba4bf44c25c6a946cb9b0f00b73044d74308b5e0afd190338297c"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:10250a093741ec7bf74bcd2039e697f519b028518f605ff2aa7ac1e9c9f97423"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6b8d0c153f07a953636b9cdb3011b733cadd4178123ef728ccc4d5969e67f3c2"}, + {file = "regex-2023.5.5-cp37-cp37m-win32.whl", hash = "sha256:10374c84ee58c44575b667310d5bbfa89fb2e64e52349720a0182c0017512f6c"}, + {file = "regex-2023.5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9b320677521aabf666cdd6e99baee4fb5ac3996349c3b7f8e7c4eee1c00dfe3a"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:afb1c70ec1e594a547f38ad6bf5e3d60304ce7539e677c1429eebab115bce56e"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf123225945aa58b3057d0fba67e8061c62d14cc8a4202630f8057df70189051"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99757ad7fe5c8a2bb44829fc57ced11253e10f462233c1255fe03888e06bc19"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a623564d810e7a953ff1357f7799c14bc9beeab699aacc8b7ab7822da1e952b8"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ced02e3bd55e16e89c08bbc8128cff0884d96e7f7a5633d3dc366b6d95fcd1d6"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cbe6b5be3b9b698d8cc4ee4dee7e017ad655e83361cd0ea8e653d65e469468"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a6e4b0e0531223f53bad07ddf733af490ba2b8367f62342b92b39b29f72735a"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e9c4f778514a560a9c9aa8e5538bee759b55f6c1dcd35613ad72523fd9175b8"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:256f7f4c6ba145f62f7a441a003c94b8b1af78cee2cccacfc1e835f93bc09426"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd7b68fd2e79d59d86dcbc1ccd6e2ca09c505343445daaa4e07f43c8a9cc34da"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4a5059bd585e9e9504ef9c07e4bc15b0a621ba20504388875d66b8b30a5c4d18"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:6893544e06bae009916a5658ce7207e26ed17385149f35a3125f5259951f1bbe"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c64d5abe91a3dfe5ff250c6bb267ef00dbc01501518225b45a5f9def458f31fb"}, + {file = "regex-2023.5.5-cp38-cp38-win32.whl", hash = "sha256:7923470d6056a9590247ff729c05e8e0f06bbd4efa6569c916943cb2d9b68b91"}, + {file = "regex-2023.5.5-cp38-cp38-win_amd64.whl", hash = "sha256:4035d6945cb961c90c3e1c1ca2feb526175bcfed44dfb1cc77db4fdced060d3e"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50fd2d9b36938d4dcecbd684777dd12a407add4f9f934f235c66372e630772b0"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d19e57f888b00cd04fc38f5e18d0efbd91ccba2d45039453ab2236e6eec48d4d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd966475e963122ee0a7118ec9024388c602d12ac72860f6eea119a3928be053"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db09e6c18977a33fea26fe67b7a842f706c67cf8bda1450974d0ae0dd63570df"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6164d4e2a82f9ebd7752a06bd6c504791bedc6418c0196cd0a23afb7f3e12b2d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84397d3f750d153ebd7f958efaa92b45fea170200e2df5e0e1fd4d85b7e3f58a"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c3efee9bb53cbe7b285760c81f28ac80dc15fa48b5fe7e58b52752e642553f1"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:144b5b017646b5a9392a5554a1e5db0000ae637be4971c9747566775fc96e1b2"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1189fbbb21e2c117fda5303653b61905aeeeea23de4a94d400b0487eb16d2d60"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f83fe9e10f9d0b6cf580564d4d23845b9d692e4c91bd8be57733958e4c602956"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:72aa4746993a28c841e05889f3f1b1e5d14df8d3daa157d6001a34c98102b393"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:de2f780c3242ea114dd01f84848655356af4dd561501896c751d7b885ea6d3a1"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:290fd35219486dfbc00b0de72f455ecdd63e59b528991a6aec9fdfc0ce85672e"}, + {file = "regex-2023.5.5-cp39-cp39-win32.whl", hash = "sha256:732176f5427e72fa2325b05c58ad0b45af341c459910d766f814b0584ac1f9ac"}, + {file = "regex-2023.5.5-cp39-cp39-win_amd64.whl", hash = "sha256:1307aa4daa1cbb23823d8238e1f61292fd07e4e5d8d38a6efff00b67a7cdb764"}, + {file = "regex-2023.5.5.tar.gz", hash = "sha256:7d76a8a1fc9da08296462a18f16620ba73bcbf5909e42383b253ef34d9d5141e"}, +] + [[package]] name = "requests" version = "2.28.2" @@ -1268,6 +1339,72 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "ruamel-yaml" +version = "0.17.31" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +category = "main" +optional = false +python-versions = ">=3" +files = [ + {file = "ruamel.yaml-0.17.31-py3-none-any.whl", hash = "sha256:3cf153f0047ced526e723097ac615d3009371779432e304dbd5596b6f3a4c777"}, + {file = "ruamel.yaml-0.17.31.tar.gz", hash = "sha256:098ed1eb6d338a684891a72380277c1e6fc4d4ae0e120de9a447275056dda335"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.12\""} + +[package.extras] +docs = ["ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.7" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, + {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, + {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, + {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, + {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, + {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, + {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, +] + [[package]] name = "s3transfer" version = "0.6.0" @@ -1457,4 +1594,4 @@ test = ["docker"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "c1e54c3c6cb146197e2bb37013bf758b9dd25bcb3c8dfc48c553823478264101" +content-hash = "a4295ac23e667931d5858408ee67bcffa9cb58d1520b794a4e544a9b6a15f6eb" diff --git a/pyproject.toml b/pyproject.toml index 8911155eb..97d8487ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,8 @@ packaging = "^23.0" typing-extensions = "^4.0.0" gojsonnet = { version = "^0.17.0", optional = true } docker = { version = "^5.0.0", optional = true } -omegaconf = "^2.3.0" +ruamel-yaml = "^0.17.31" +regex = "^2023.5.5" [tool.poetry.extras] gojsonnet = ["gojsonnet"] From 3b5c8db10a6c0e11978c7d3f65c1768e7726b3c7 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:45:31 +0200 Subject: [PATCH 13/37] feat: prepare omegaconf for multiprocessing --- kapitan/omegaconf.py | 126 +++++++++++++++++++++++++++++++++++++++++++ kapitan/resources.py | 105 +++--------------------------------- 2 files changed, 134 insertions(+), 97 deletions(-) create mode 100644 kapitan/omegaconf.py diff --git a/kapitan/omegaconf.py b/kapitan/omegaconf.py new file mode 100644 index 000000000..70efaf1a2 --- /dev/null +++ b/kapitan/omegaconf.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# Copyright 2023 nexenio +import logging +import multiprocessing +import os + +from kapitan.errors import InventoryError +from omegaconf import Node, OmegaConf, errors + +logger = logging.getLogger(__name__) + + +def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], compose_node_name=False): + """ + generates inventory from yaml files using OmegaConf + """ + # TBD: add config to specify paths (do we need that?) + targets_searchpath = os.path.join(inventory_path, "targets") + classes_searchpath = os.path.join(inventory_path, "classes") + + def key(_node_: Node): + """resolver function, that returns the name of its parent key""" + return _node_._key() + + def fullkey(_node_: Node): + """resolver function, that returns the full name of its parent key""" + return _node_._get_full_key("") + + def dep(name: str): + """resolver function, that returns a parameterized dependency""" + return f"dependency{name}" + + OmegaConf.register_new_resolver("key", key) + OmegaConf.register_new_resolver("fullkey", fullkey) + + # kapitan helpers + OmegaConf.register_new_resolver("dep", dep) + selected_targets = [] + + # loop through targets searchpath and load all targets + for root, dirs, files in os.walk(targets_searchpath): + for target_name in files: + target_path = os.path.join(root, target_name) + + # split file extension and check if yml/yaml + target_name, ext = os.path.splitext(target_name) + if ext not in (".yml", ".yaml"): + logger.debug(f"{target_name}: targets have to be .yml or .yaml files.") + # RAISE ERROR + continue + + # skip targets if they are not specified with -t flag + if targets and target_name not in targets: + continue + + # compose node name + if compose_node_name: + target_name = str(os.path.splitext(target_path)[0]).replace(targets_searchpath + os.sep, "") + target_name = target_name.replace("/", ".") + + selected_targets.append({"name": target_name, "path": target_path}) + + inv = {"nodes": {}} + for target in selected_targets: + inv["nodes"][target["name"]] = load_target( + target["name"], target["path"], classes_searchpath, ignore_class_notfound + ) + + # TBD: refactor inventory accessing (targets.py, cmd_parser.py) + # that it only receives the targets and not everything + return inv + + +def load_target( + target_name: str, target_path: str, classes_searchpath: str, ignore_class_notfound: bool = False +): + # load the target + target_config = OmegaConf.load(target_path) + + target_config_classes = target_config.pop("classes", []) + + # load classes for targets + for class_name in target_config_classes: + # resolve class name (relative paths TBD) + class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" + if os.path.isfile(class_path): + # load classes recursively + class_config = OmegaConf.load(class_path) + target_config_classes.extend(class_config.pop("classes", [])) + + elif not ignore_class_notfound: + raise InventoryError(f"{target_name}: Class {class_name} not found.") + else: + continue + + # merge target with loaded classes + if target_config.get("parameters"): + target_config = OmegaConf.merge(class_config, target_config, extend_lists=True) + else: + target_config = class_config + + if not target_config: + raise InventoryError(f"{target_name}: empty target") + + if not target_config.get("parameters"): + raise InventoryError(f"{target_name}: target has no parameters") + + # append meta data _reclass_ (legacy) (refactoring TBD) + target_config["parameters"]["_reclass_"] = { + "name": {"full": target_name, "path": target_name, "short": target_name} + } + + # resolve references / interpolate values + try: + target_config = OmegaConf.to_container(target_config, resolve=True) + except errors.OmegaConfBaseException as e: + raise InventoryError(f"{target_name}: {e.__context__}") + + # obtain target name to insert in inv dict (legacy) (refactoring TBD) + try: + target_name = target_config["parameters"]["kapitan"]["vars"]["target"] + except KeyError: + logger.warning(f"Could not resolve target name on target {target_name}") + + return target_config diff --git a/kapitan/resources.py b/kapitan/resources.py index 0a2499f99..8fe05094a 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -17,17 +17,18 @@ from functools import partial import jsonschema -import kapitan.cached as cached +import reclass +import reclass.core import yaml +from reclass.errors import NotFoundError, ReclassException + +import kapitan.cached as cached from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError -from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string from kapitan.migrate_omegaconf import migrate - -import reclass -import reclass.core -from reclass.errors import NotFoundError, ReclassException -from omegaconf import OmegaConf, errors +from kapitan.omegaconf import inventory_omegaconf +from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string +from omegaconf import errors logger = logging.getLogger(__name__) @@ -407,93 +408,3 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): else: logger.error("Inventory reclass error: %s", e.message) raise InventoryError(e.message) - - -def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], compose_node_name=False): - """ - generates inventory from yaml files using OmegaConf - """ - # TBD: add config to specify paths (do we need that?) - targets_searchpath = os.path.join(inventory_path, "targets") - classes_searchpath = os.path.join(inventory_path, "classes") - - inv = {"nodes": {}} - - # loop through targets searchpath and load all targets - for root, dirs, files in os.walk(targets_searchpath): - for target_name in files: - target_path = os.path.join(root, target_name) - - # split file extension and check if yml/yaml - target_name, ext = os.path.splitext(target_name) - if ext not in (".yml", ".yaml"): - logger.debug(f"{target_name}: targets have to be .yml or .yaml files.") - continue - - # skip targets if they are not specified with -t flag - if targets and target_name not in targets: - continue - - # compose node name - if compose_node_name: - target_name = str(os.path.splitext(target_path)[0]).replace(targets_searchpath + os.sep, "") - target_name = target_name.replace("/", ".") - - # look for duplicate targets - if target_name in inv["nodes"]: - raise InventoryError(f"{target_name}: duplicate target are not allowed") - - # load the inventory - target_config = OmegaConf.load(target_path) - - target_config_classes = target_config.pop("classes", []) - - # load classes for targets - for class_name in target_config_classes: - # resolve class name (relative paths TBD) - class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" - if os.path.isfile(class_path): - # load classes recursively - class_config = OmegaConf.load(class_path) - - target_config_classes.extend(class_config.pop("classes", [])) - - elif not ignore_class_notfound: - raise InventoryError(f"{target_name}: Class {class_name} not found.") - else: - continue - - # merge target with loaded classes - if target_config.get("parameters"): - target_config = OmegaConf.merge(class_config, target_config, extend_lists=True) - else: - target_config = class_config - - if not target_config: - raise InventoryError(f"{target_name}: empty target") - - if not target_config.get("parameters"): - raise InventoryError(f"{target_name}: target has no parameters") - - # append meta data _reclass_ (legacy) (refactoring TBD) - target_config["parameters"]["_reclass_"] = { - "name": {"full": target_name, "path": target_name, "short": target_name} - } - - # resolve references / interpolate values - try: - target_config = OmegaConf.to_container(target_config, resolve=True) - except errors.OmegaConfBaseException as e: - raise InventoryError(f"{target_name}: {e.__context__}") - - # obtain target name to insert in inv dict (legacy) (refactoring TBD) - try: - target_name = target_config["parameters"]["kapitan"]["vars"]["target"] - except KeyError: - logger.warning(f"Could not resolve target name on target {target_name}") - - inv["nodes"][target_name] = target_config - - # TBD: refactor inventory accessing (targets.py, cmd_parser.py) - # that it only receives the targets and not everything - return inv From d20be197f461aa24a0ad6aca6e2cd0e39853296d Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 8 Jun 2023 11:25:20 +0200 Subject: [PATCH 14/37] feat: add generated grammar --- kapitan/migrate_omegaconf.py | 26 +- kapitan/omegaconf.py | 49 +- .../grammar/gen/OmegaConfGrammarLexer.interp | 130 ++ .../grammar/gen/OmegaConfGrammarLexer.py | 404 ++++ .../grammar/gen/OmegaConfGrammarLexer.tokens | 31 + .../grammar/gen/OmegaConfGrammarParser.interp | 83 + .../grammar/gen/OmegaConfGrammarParser.py | 1745 +++++++++++++++++ .../grammar/gen/OmegaConfGrammarParser.tokens | 31 + .../gen/OmegaConfGrammarParserListener.py | 141 ++ .../gen/OmegaConfGrammarParserVisitor.py | 78 + 10 files changed, 2703 insertions(+), 15 deletions(-) create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.interp create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.py create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.interp create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.py create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.tokens create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParserListener.py create mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py index 8942ffb26..04f157469 100644 --- a/kapitan/migrate_omegaconf.py +++ b/kapitan/migrate_omegaconf.py @@ -6,7 +6,7 @@ from pathlib import Path import regex as re -REF_TOKEN = r"\${([^\${}]*+(?:(?R)[^\${}]*)*+)}" +REF_TOKEN = r"(? str: @@ -64,27 +64,41 @@ def migrate_file(input_file: str) -> None: file_path = Path(input_file).resolve() - yaml_obj = yaml.load(file_path) - yaml.dump(yaml_obj, file_path) - - yaml_obj = migrate_yaml_obj(yaml_obj) + try: + yaml_obj = yaml.load(file_path) + yaml_obj = migrate_yaml_obj(yaml_obj) + except: + print("ERROR in: ", file_path) + return + # yaml.dump(yaml_obj, file_path) yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(yaml_obj, file_path) -def migrate(inv_path: str = "", output_path: str = "") -> None: +def migrate(inv_path: str, output_path: str = "") -> None: targets_path = os.path.join(inv_path, "targets") classes_path = os.path.join(inv_path, "classes") for root, subdirs, files in os.walk(targets_path): for target_file in files: target_file = os.path.join(root, target_file) + _, ext = os.path.splitext(target_file) + + if ext not in (".yml", ".yaml"): + continue + migrate_file(target_file) for root, subdirs, files in os.walk(classes_path): for class_file in files: class_file = os.path.join(root, class_file) + + _, ext = os.path.splitext(class_file) + + if ext not in (".yml", ".yaml"): + continue + migrate_file(class_file) diff --git a/kapitan/omegaconf.py b/kapitan/omegaconf.py index 70efaf1a2..ebafd09ef 100644 --- a/kapitan/omegaconf.py +++ b/kapitan/omegaconf.py @@ -2,8 +2,10 @@ # Copyright 2023 nexenio import logging -import multiprocessing +from multiprocessing.pool import ThreadPool +from functools import partial import os +import time from kapitan.errors import InventoryError from omegaconf import Node, OmegaConf, errors @@ -62,21 +64,47 @@ def dep(name: str): selected_targets.append({"name": target_name, "path": target_path}) inv = {"nodes": {}} + # load targets parallel + + stamp = time.time() for target in selected_targets: - inv["nodes"][target["name"]] = load_target( - target["name"], target["path"], classes_searchpath, ignore_class_notfound - ) + try: + name, config = load_target(target, classes_searchpath, ignore_class_notfound) + inv["nodes"][name] = config + except Exception as e: + print(target, e) + + # pool = ThreadPool(8) + + # worker = partial( + # load_target, + # classes_searchpath=classes_searchpath, + # ignore_class_notfound=ignore_class_notfound, + # ) + + # for p in pool.imap(worker, selected_targets): + # name, config = p + # inv["nodes"][name] = config + + # pool.close() + + print(f"real time: {time.time() - stamp}") # TBD: refactor inventory accessing (targets.py, cmd_parser.py) # that it only receives the targets and not everything return inv -def load_target( - target_name: str, target_path: str, classes_searchpath: str, ignore_class_notfound: bool = False -): +def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bool = False): # load the target - target_config = OmegaConf.load(target_path) + target_name = target["name"] + target_path = target["path"] + + try: + target_config = OmegaConf.load(target_path) + except: + print(target) + return "", {} target_config_classes = target_config.pop("classes", []) @@ -123,4 +151,7 @@ def load_target( except KeyError: logger.warning(f"Could not resolve target name on target {target_name}") - return target_config + # # stamp = time.time() + # logger.info("loaded %s in %.4f", target_name, 1000*(time.time() - stamp)) + + return target_name, target_config diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp b/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp new file mode 100644 index 000000000..98152df6e --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp @@ -0,0 +1,130 @@ +token literal names: +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +'.' +null +null +null +null +'[' +']' + +token symbolic names: +null +ANY_STR +ESC_INTER +TOP_ESC +INTER_OPEN +BRACE_OPEN +BRACE_CLOSE +QUOTE_OPEN_SINGLE +QUOTE_OPEN_DOUBLE +COMMA +BRACKET_OPEN +BRACKET_CLOSE +COLON +FLOAT +INT +BOOL +NULL +UNQUOTED_CHAR +ID +ESC +WS +INTER_CLOSE +DOT +INTER_KEY +MATCHING_QUOTE_CLOSE +QUOTED_ESC +DOLLAR +INTER_BRACKET_OPEN +INTER_BRACKET_CLOSE + +rule names: +CHAR +DIGIT +INT_UNSIGNED +ESC_BACKSLASH +TOP_INTER_OPEN +ANY_STR +ESC_INTER +TOP_ESC +BACKSLASHES +DOLLAR +INTER_OPEN +BRACE_OPEN +BRACE_CLOSE +QUOTE_OPEN_SINGLE +QUOTE_OPEN_DOUBLE +COMMA +BRACKET_OPEN +BRACKET_CLOSE +COLON +POINT_FLOAT +EXPONENT_FLOAT +FLOAT +INT +BOOL +NULL +UNQUOTED_CHAR +ID +ESC +WS +NESTED_INTER_OPEN +INTER_COLON +INTER_CLOSE +DOT +INTER_BRACKET_OPEN +INTER_BRACKET_CLOSE +INTER_ID +INTER_KEY +QSINGLE_INTER_OPEN +MATCHING_QUOTE_CLOSE +QSINGLE_STR +QSINGLE_ESC_INTER +QSINGLE_ESC_QUOTE +QUOTED_ESC +QSINGLE_BACKSLASHES +QSINGLE_DOLLAR +QDOUBLE_INTER_OPEN +QDOUBLE_CLOSE +QDOUBLE_STR +QDOUBLE_ESC_INTER +QDOUBLE_ESC_QUOTE +QDOUBLE_ESC +QDOUBLE_BACKSLASHES +QDOUBLE_DOLLAR + +channel names: +DEFAULT_TOKEN_CHANNEL +HIDDEN + +mode names: +DEFAULT_MODE +VALUE_MODE +INTERPOLATION_MODE +QUOTED_SINGLE_MODE +QUOTED_DOUBLE_MODE + +atn: +[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 30, 487, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 4, 35, 9, 35, 4, 36, 9, 36, 4, 37, 9, 37, 4, 38, 9, 38, 4, 39, 9, 39, 4, 40, 9, 40, 4, 41, 9, 41, 4, 42, 9, 42, 4, 43, 9, 43, 4, 44, 9, 44, 4, 45, 9, 45, 4, 46, 9, 46, 4, 47, 9, 47, 4, 48, 9, 48, 4, 49, 9, 49, 4, 50, 9, 50, 4, 51, 9, 51, 4, 52, 9, 52, 4, 53, 9, 53, 4, 54, 9, 54, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 5, 4, 121, 10, 4, 3, 4, 7, 4, 124, 10, 4, 12, 4, 14, 4, 127, 11, 4, 5, 4, 129, 10, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 7, 7, 7, 140, 10, 7, 12, 7, 14, 7, 143, 11, 7, 3, 7, 3, 7, 3, 8, 7, 8, 148, 10, 8, 12, 8, 14, 8, 151, 11, 8, 3, 8, 3, 8, 3, 8, 3, 8, 3, 9, 6, 9, 158, 10, 9, 13, 9, 14, 9, 159, 3, 10, 6, 10, 163, 10, 10, 13, 10, 14, 10, 164, 3, 10, 3, 10, 3, 11, 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 5, 12, 177, 10, 12, 3, 12, 3, 12, 3, 13, 3, 13, 5, 13, 183, 10, 13, 3, 13, 3, 13, 3, 14, 5, 14, 188, 10, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 15, 3, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 17, 5, 17, 203, 10, 17, 3, 17, 3, 17, 5, 17, 207, 10, 17, 3, 18, 3, 18, 5, 18, 211, 10, 18, 3, 19, 5, 19, 214, 10, 19, 3, 19, 3, 19, 3, 20, 5, 20, 219, 10, 20, 3, 20, 3, 20, 5, 20, 223, 10, 20, 3, 21, 3, 21, 3, 21, 3, 21, 5, 21, 229, 10, 21, 3, 21, 3, 21, 3, 21, 5, 21, 234, 10, 21, 3, 21, 7, 21, 237, 10, 21, 12, 21, 14, 21, 240, 11, 21, 5, 21, 242, 10, 21, 3, 22, 3, 22, 5, 22, 246, 10, 22, 3, 22, 3, 22, 5, 22, 250, 10, 22, 3, 22, 3, 22, 5, 22, 254, 10, 22, 3, 22, 7, 22, 257, 10, 22, 12, 22, 14, 22, 260, 11, 22, 3, 23, 5, 23, 263, 10, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 273, 10, 23, 3, 24, 5, 24, 276, 10, 24, 3, 24, 3, 24, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 5, 25, 289, 10, 25, 3, 26, 3, 26, 3, 26, 3, 26, 3, 26, 3, 27, 3, 27, 3, 28, 3, 28, 5, 28, 300, 10, 28, 3, 28, 3, 28, 3, 28, 7, 28, 305, 10, 28, 12, 28, 14, 28, 308, 11, 28, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 6, 29, 333, 10, 29, 13, 29, 14, 29, 334, 3, 30, 6, 30, 338, 10, 30, 13, 30, 14, 30, 339, 3, 31, 3, 31, 5, 31, 344, 10, 31, 3, 31, 3, 31, 3, 31, 3, 32, 5, 32, 350, 10, 32, 3, 32, 3, 32, 5, 32, 354, 10, 32, 3, 32, 3, 32, 3, 32, 3, 33, 5, 33, 360, 10, 33, 3, 33, 3, 33, 3, 33, 3, 33, 3, 34, 3, 34, 3, 35, 3, 35, 3, 35, 3, 35, 3, 36, 3, 36, 3, 36, 3, 36, 3, 37, 3, 37, 3, 37, 3, 37, 3, 38, 6, 38, 381, 10, 38, 13, 38, 14, 38, 382, 3, 39, 3, 39, 3, 39, 3, 39, 3, 39, 3, 40, 3, 40, 3, 40, 3, 40, 3, 41, 7, 41, 395, 10, 41, 12, 41, 14, 41, 398, 11, 41, 3, 41, 3, 41, 3, 41, 3, 41, 3, 42, 3, 42, 3, 42, 3, 42, 3, 43, 7, 43, 409, 10, 43, 12, 43, 14, 43, 412, 11, 43, 3, 43, 3, 43, 3, 43, 3, 43, 3, 43, 3, 44, 6, 44, 420, 10, 44, 13, 44, 14, 44, 421, 3, 45, 6, 45, 425, 10, 45, 13, 45, 14, 45, 426, 3, 45, 3, 45, 3, 46, 3, 46, 3, 46, 3, 46, 3, 47, 3, 47, 3, 47, 3, 47, 3, 47, 3, 48, 3, 48, 3, 48, 3, 48, 3, 48, 3, 49, 7, 49, 446, 10, 49, 12, 49, 14, 49, 449, 11, 49, 3, 49, 3, 49, 3, 49, 3, 49, 3, 50, 3, 50, 3, 50, 3, 50, 3, 51, 7, 51, 460, 10, 51, 12, 51, 14, 51, 463, 11, 51, 3, 51, 3, 51, 3, 51, 3, 51, 3, 51, 3, 52, 6, 52, 471, 10, 52, 13, 52, 14, 52, 472, 3, 52, 3, 52, 3, 53, 6, 53, 478, 10, 53, 13, 53, 14, 53, 479, 3, 53, 3, 53, 3, 54, 3, 54, 3, 54, 3, 54, 2, 2, 55, 7, 2, 9, 2, 11, 2, 13, 2, 15, 2, 17, 3, 19, 4, 21, 5, 23, 2, 25, 28, 27, 6, 29, 7, 31, 8, 33, 9, 35, 10, 37, 11, 39, 12, 41, 13, 43, 14, 45, 2, 47, 2, 49, 15, 51, 16, 53, 17, 55, 18, 57, 19, 59, 20, 61, 21, 63, 22, 65, 2, 67, 2, 69, 23, 71, 24, 73, 29, 75, 30, 77, 2, 79, 25, 81, 2, 83, 26, 85, 2, 87, 2, 89, 2, 91, 27, 93, 2, 95, 2, 97, 2, 99, 2, 101, 2, 103, 2, 105, 2, 107, 2, 109, 2, 111, 2, 7, 2, 3, 4, 5, 6, 26, 4, 2, 67, 92, 99, 124, 3, 2, 50, 59, 3, 2, 51, 59, 3, 2, 38, 38, 4, 2, 38, 38, 94, 94, 4, 2, 71, 71, 103, 103, 4, 2, 45, 45, 47, 47, 4, 2, 75, 75, 107, 107, 4, 2, 80, 80, 112, 112, 4, 2, 72, 72, 104, 104, 4, 2, 67, 67, 99, 99, 4, 2, 86, 86, 118, 118, 4, 2, 84, 84, 116, 116, 4, 2, 87, 87, 119, 119, 4, 2, 78, 78, 110, 110, 4, 2, 85, 85, 117, 117, 8, 2, 38, 39, 44, 45, 47, 49, 65, 66, 94, 94, 126, 126, 4, 2, 47, 47, 97, 97, 4, 2, 11, 11, 34, 34, 11, 2, 11, 11, 34, 34, 36, 36, 41, 43, 48, 48, 60, 60, 93, 95, 125, 125, 127, 127, 4, 2, 38, 38, 41, 41, 5, 2, 38, 38, 41, 41, 94, 94, 4, 2, 36, 36, 38, 38, 5, 2, 36, 36, 38, 38, 94, 94, 2, 536, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 3, 27, 3, 2, 2, 2, 3, 29, 3, 2, 2, 2, 3, 31, 3, 2, 2, 2, 3, 33, 3, 2, 2, 2, 3, 35, 3, 2, 2, 2, 3, 37, 3, 2, 2, 2, 3, 39, 3, 2, 2, 2, 3, 41, 3, 2, 2, 2, 3, 43, 3, 2, 2, 2, 3, 49, 3, 2, 2, 2, 3, 51, 3, 2, 2, 2, 3, 53, 3, 2, 2, 2, 3, 55, 3, 2, 2, 2, 3, 57, 3, 2, 2, 2, 3, 59, 3, 2, 2, 2, 3, 61, 3, 2, 2, 2, 3, 63, 3, 2, 2, 2, 4, 65, 3, 2, 2, 2, 4, 67, 3, 2, 2, 2, 4, 69, 3, 2, 2, 2, 4, 71, 3, 2, 2, 2, 4, 73, 3, 2, 2, 2, 4, 75, 3, 2, 2, 2, 4, 77, 3, 2, 2, 2, 4, 79, 3, 2, 2, 2, 5, 81, 3, 2, 2, 2, 5, 83, 3, 2, 2, 2, 5, 85, 3, 2, 2, 2, 5, 87, 3, 2, 2, 2, 5, 89, 3, 2, 2, 2, 5, 91, 3, 2, 2, 2, 5, 93, 3, 2, 2, 2, 5, 95, 3, 2, 2, 2, 6, 97, 3, 2, 2, 2, 6, 99, 3, 2, 2, 2, 6, 101, 3, 2, 2, 2, 6, 103, 3, 2, 2, 2, 6, 105, 3, 2, 2, 2, 6, 107, 3, 2, 2, 2, 6, 109, 3, 2, 2, 2, 6, 111, 3, 2, 2, 2, 7, 113, 3, 2, 2, 2, 9, 115, 3, 2, 2, 2, 11, 128, 3, 2, 2, 2, 13, 130, 3, 2, 2, 2, 15, 133, 3, 2, 2, 2, 17, 141, 3, 2, 2, 2, 19, 149, 3, 2, 2, 2, 21, 157, 3, 2, 2, 2, 23, 162, 3, 2, 2, 2, 25, 168, 3, 2, 2, 2, 27, 172, 3, 2, 2, 2, 29, 180, 3, 2, 2, 2, 31, 187, 3, 2, 2, 2, 33, 193, 3, 2, 2, 2, 35, 197, 3, 2, 2, 2, 37, 202, 3, 2, 2, 2, 39, 208, 3, 2, 2, 2, 41, 213, 3, 2, 2, 2, 43, 218, 3, 2, 2, 2, 45, 241, 3, 2, 2, 2, 47, 245, 3, 2, 2, 2, 49, 262, 3, 2, 2, 2, 51, 275, 3, 2, 2, 2, 53, 288, 3, 2, 2, 2, 55, 290, 3, 2, 2, 2, 57, 295, 3, 2, 2, 2, 59, 299, 3, 2, 2, 2, 61, 332, 3, 2, 2, 2, 63, 337, 3, 2, 2, 2, 65, 341, 3, 2, 2, 2, 67, 349, 3, 2, 2, 2, 69, 359, 3, 2, 2, 2, 71, 365, 3, 2, 2, 2, 73, 367, 3, 2, 2, 2, 75, 371, 3, 2, 2, 2, 77, 375, 3, 2, 2, 2, 79, 380, 3, 2, 2, 2, 81, 384, 3, 2, 2, 2, 83, 389, 3, 2, 2, 2, 85, 396, 3, 2, 2, 2, 87, 403, 3, 2, 2, 2, 89, 410, 3, 2, 2, 2, 91, 419, 3, 2, 2, 2, 93, 424, 3, 2, 2, 2, 95, 430, 3, 2, 2, 2, 97, 434, 3, 2, 2, 2, 99, 439, 3, 2, 2, 2, 101, 447, 3, 2, 2, 2, 103, 454, 3, 2, 2, 2, 105, 461, 3, 2, 2, 2, 107, 470, 3, 2, 2, 2, 109, 477, 3, 2, 2, 2, 111, 483, 3, 2, 2, 2, 113, 114, 9, 2, 2, 2, 114, 8, 3, 2, 2, 2, 115, 116, 9, 3, 2, 2, 116, 10, 3, 2, 2, 2, 117, 129, 7, 50, 2, 2, 118, 125, 9, 4, 2, 2, 119, 121, 7, 97, 2, 2, 120, 119, 3, 2, 2, 2, 120, 121, 3, 2, 2, 2, 121, 122, 3, 2, 2, 2, 122, 124, 5, 9, 3, 2, 123, 120, 3, 2, 2, 2, 124, 127, 3, 2, 2, 2, 125, 123, 3, 2, 2, 2, 125, 126, 3, 2, 2, 2, 126, 129, 3, 2, 2, 2, 127, 125, 3, 2, 2, 2, 128, 117, 3, 2, 2, 2, 128, 118, 3, 2, 2, 2, 129, 12, 3, 2, 2, 2, 130, 131, 7, 94, 2, 2, 131, 132, 7, 94, 2, 2, 132, 14, 3, 2, 2, 2, 133, 134, 5, 27, 12, 2, 134, 135, 3, 2, 2, 2, 135, 136, 8, 6, 2, 2, 136, 137, 8, 6, 3, 2, 137, 16, 3, 2, 2, 2, 138, 140, 10, 5, 2, 2, 139, 138, 3, 2, 2, 2, 140, 143, 3, 2, 2, 2, 141, 139, 3, 2, 2, 2, 141, 142, 3, 2, 2, 2, 142, 144, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 144, 145, 10, 6, 2, 2, 145, 18, 3, 2, 2, 2, 146, 148, 5, 13, 5, 2, 147, 146, 3, 2, 2, 2, 148, 151, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 152, 3, 2, 2, 2, 151, 149, 3, 2, 2, 2, 152, 153, 7, 94, 2, 2, 153, 154, 7, 38, 2, 2, 154, 155, 7, 125, 2, 2, 155, 20, 3, 2, 2, 2, 156, 158, 5, 13, 5, 2, 157, 156, 3, 2, 2, 2, 158, 159, 3, 2, 2, 2, 159, 157, 3, 2, 2, 2, 159, 160, 3, 2, 2, 2, 160, 22, 3, 2, 2, 2, 161, 163, 7, 94, 2, 2, 162, 161, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 162, 3, 2, 2, 2, 164, 165, 3, 2, 2, 2, 165, 166, 3, 2, 2, 2, 166, 167, 8, 10, 4, 2, 167, 24, 3, 2, 2, 2, 168, 169, 7, 38, 2, 2, 169, 170, 3, 2, 2, 2, 170, 171, 8, 11, 4, 2, 171, 26, 3, 2, 2, 2, 172, 173, 7, 38, 2, 2, 173, 174, 7, 125, 2, 2, 174, 176, 3, 2, 2, 2, 175, 177, 5, 63, 30, 2, 176, 175, 3, 2, 2, 2, 176, 177, 3, 2, 2, 2, 177, 178, 3, 2, 2, 2, 178, 179, 8, 12, 3, 2, 179, 28, 3, 2, 2, 2, 180, 182, 7, 125, 2, 2, 181, 183, 5, 63, 30, 2, 182, 181, 3, 2, 2, 2, 182, 183, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 185, 8, 13, 5, 2, 185, 30, 3, 2, 2, 2, 186, 188, 5, 63, 30, 2, 187, 186, 3, 2, 2, 2, 187, 188, 3, 2, 2, 2, 188, 189, 3, 2, 2, 2, 189, 190, 7, 127, 2, 2, 190, 191, 3, 2, 2, 2, 191, 192, 8, 14, 6, 2, 192, 32, 3, 2, 2, 2, 193, 194, 7, 41, 2, 2, 194, 195, 3, 2, 2, 2, 195, 196, 8, 15, 7, 2, 196, 34, 3, 2, 2, 2, 197, 198, 7, 36, 2, 2, 198, 199, 3, 2, 2, 2, 199, 200, 8, 16, 8, 2, 200, 36, 3, 2, 2, 2, 201, 203, 5, 63, 30, 2, 202, 201, 3, 2, 2, 2, 202, 203, 3, 2, 2, 2, 203, 204, 3, 2, 2, 2, 204, 206, 7, 46, 2, 2, 205, 207, 5, 63, 30, 2, 206, 205, 3, 2, 2, 2, 206, 207, 3, 2, 2, 2, 207, 38, 3, 2, 2, 2, 208, 210, 7, 93, 2, 2, 209, 211, 5, 63, 30, 2, 210, 209, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 40, 3, 2, 2, 2, 212, 214, 5, 63, 30, 2, 213, 212, 3, 2, 2, 2, 213, 214, 3, 2, 2, 2, 214, 215, 3, 2, 2, 2, 215, 216, 7, 95, 2, 2, 216, 42, 3, 2, 2, 2, 217, 219, 5, 63, 30, 2, 218, 217, 3, 2, 2, 2, 218, 219, 3, 2, 2, 2, 219, 220, 3, 2, 2, 2, 220, 222, 7, 60, 2, 2, 221, 223, 5, 63, 30, 2, 222, 221, 3, 2, 2, 2, 222, 223, 3, 2, 2, 2, 223, 44, 3, 2, 2, 2, 224, 225, 5, 11, 4, 2, 225, 226, 7, 48, 2, 2, 226, 242, 3, 2, 2, 2, 227, 229, 5, 11, 4, 2, 228, 227, 3, 2, 2, 2, 228, 229, 3, 2, 2, 2, 229, 230, 3, 2, 2, 2, 230, 231, 7, 48, 2, 2, 231, 238, 5, 9, 3, 2, 232, 234, 7, 97, 2, 2, 233, 232, 3, 2, 2, 2, 233, 234, 3, 2, 2, 2, 234, 235, 3, 2, 2, 2, 235, 237, 5, 9, 3, 2, 236, 233, 3, 2, 2, 2, 237, 240, 3, 2, 2, 2, 238, 236, 3, 2, 2, 2, 238, 239, 3, 2, 2, 2, 239, 242, 3, 2, 2, 2, 240, 238, 3, 2, 2, 2, 241, 224, 3, 2, 2, 2, 241, 228, 3, 2, 2, 2, 242, 46, 3, 2, 2, 2, 243, 246, 5, 11, 4, 2, 244, 246, 5, 45, 21, 2, 245, 243, 3, 2, 2, 2, 245, 244, 3, 2, 2, 2, 246, 247, 3, 2, 2, 2, 247, 249, 9, 7, 2, 2, 248, 250, 9, 8, 2, 2, 249, 248, 3, 2, 2, 2, 249, 250, 3, 2, 2, 2, 250, 251, 3, 2, 2, 2, 251, 258, 5, 9, 3, 2, 252, 254, 7, 97, 2, 2, 253, 252, 3, 2, 2, 2, 253, 254, 3, 2, 2, 2, 254, 255, 3, 2, 2, 2, 255, 257, 5, 9, 3, 2, 256, 253, 3, 2, 2, 2, 257, 260, 3, 2, 2, 2, 258, 256, 3, 2, 2, 2, 258, 259, 3, 2, 2, 2, 259, 48, 3, 2, 2, 2, 260, 258, 3, 2, 2, 2, 261, 263, 9, 8, 2, 2, 262, 261, 3, 2, 2, 2, 262, 263, 3, 2, 2, 2, 263, 272, 3, 2, 2, 2, 264, 273, 5, 45, 21, 2, 265, 273, 5, 47, 22, 2, 266, 267, 9, 9, 2, 2, 267, 268, 9, 10, 2, 2, 268, 273, 9, 11, 2, 2, 269, 270, 9, 10, 2, 2, 270, 271, 9, 12, 2, 2, 271, 273, 9, 10, 2, 2, 272, 264, 3, 2, 2, 2, 272, 265, 3, 2, 2, 2, 272, 266, 3, 2, 2, 2, 272, 269, 3, 2, 2, 2, 273, 50, 3, 2, 2, 2, 274, 276, 9, 8, 2, 2, 275, 274, 3, 2, 2, 2, 275, 276, 3, 2, 2, 2, 276, 277, 3, 2, 2, 2, 277, 278, 5, 11, 4, 2, 278, 52, 3, 2, 2, 2, 279, 280, 9, 13, 2, 2, 280, 281, 9, 14, 2, 2, 281, 282, 9, 15, 2, 2, 282, 289, 9, 7, 2, 2, 283, 284, 9, 11, 2, 2, 284, 285, 9, 12, 2, 2, 285, 286, 9, 16, 2, 2, 286, 287, 9, 17, 2, 2, 287, 289, 9, 7, 2, 2, 288, 279, 3, 2, 2, 2, 288, 283, 3, 2, 2, 2, 289, 54, 3, 2, 2, 2, 290, 291, 9, 10, 2, 2, 291, 292, 9, 15, 2, 2, 292, 293, 9, 16, 2, 2, 293, 294, 9, 16, 2, 2, 294, 56, 3, 2, 2, 2, 295, 296, 9, 18, 2, 2, 296, 58, 3, 2, 2, 2, 297, 300, 5, 7, 2, 2, 298, 300, 7, 97, 2, 2, 299, 297, 3, 2, 2, 2, 299, 298, 3, 2, 2, 2, 300, 306, 3, 2, 2, 2, 301, 305, 5, 7, 2, 2, 302, 305, 5, 9, 3, 2, 303, 305, 9, 19, 2, 2, 304, 301, 3, 2, 2, 2, 304, 302, 3, 2, 2, 2, 304, 303, 3, 2, 2, 2, 305, 308, 3, 2, 2, 2, 306, 304, 3, 2, 2, 2, 306, 307, 3, 2, 2, 2, 307, 60, 3, 2, 2, 2, 308, 306, 3, 2, 2, 2, 309, 333, 5, 13, 5, 2, 310, 311, 7, 94, 2, 2, 311, 333, 7, 42, 2, 2, 312, 313, 7, 94, 2, 2, 313, 333, 7, 43, 2, 2, 314, 315, 7, 94, 2, 2, 315, 333, 7, 93, 2, 2, 316, 317, 7, 94, 2, 2, 317, 333, 7, 95, 2, 2, 318, 319, 7, 94, 2, 2, 319, 333, 7, 125, 2, 2, 320, 321, 7, 94, 2, 2, 321, 333, 7, 127, 2, 2, 322, 323, 7, 94, 2, 2, 323, 333, 7, 60, 2, 2, 324, 325, 7, 94, 2, 2, 325, 333, 7, 63, 2, 2, 326, 327, 7, 94, 2, 2, 327, 333, 7, 46, 2, 2, 328, 329, 7, 94, 2, 2, 329, 333, 7, 34, 2, 2, 330, 331, 7, 94, 2, 2, 331, 333, 7, 11, 2, 2, 332, 309, 3, 2, 2, 2, 332, 310, 3, 2, 2, 2, 332, 312, 3, 2, 2, 2, 332, 314, 3, 2, 2, 2, 332, 316, 3, 2, 2, 2, 332, 318, 3, 2, 2, 2, 332, 320, 3, 2, 2, 2, 332, 322, 3, 2, 2, 2, 332, 324, 3, 2, 2, 2, 332, 326, 3, 2, 2, 2, 332, 328, 3, 2, 2, 2, 332, 330, 3, 2, 2, 2, 333, 334, 3, 2, 2, 2, 334, 332, 3, 2, 2, 2, 334, 335, 3, 2, 2, 2, 335, 62, 3, 2, 2, 2, 336, 338, 9, 20, 2, 2, 337, 336, 3, 2, 2, 2, 338, 339, 3, 2, 2, 2, 339, 337, 3, 2, 2, 2, 339, 340, 3, 2, 2, 2, 340, 64, 3, 2, 2, 2, 341, 343, 5, 27, 12, 2, 342, 344, 5, 63, 30, 2, 343, 342, 3, 2, 2, 2, 343, 344, 3, 2, 2, 2, 344, 345, 3, 2, 2, 2, 345, 346, 8, 31, 2, 2, 346, 347, 8, 31, 3, 2, 347, 66, 3, 2, 2, 2, 348, 350, 5, 63, 30, 2, 349, 348, 3, 2, 2, 2, 349, 350, 3, 2, 2, 2, 350, 351, 3, 2, 2, 2, 351, 353, 7, 60, 2, 2, 352, 354, 5, 63, 30, 2, 353, 352, 3, 2, 2, 2, 353, 354, 3, 2, 2, 2, 354, 355, 3, 2, 2, 2, 355, 356, 8, 32, 9, 2, 356, 357, 8, 32, 10, 2, 357, 68, 3, 2, 2, 2, 358, 360, 5, 63, 30, 2, 359, 358, 3, 2, 2, 2, 359, 360, 3, 2, 2, 2, 360, 361, 3, 2, 2, 2, 361, 362, 7, 127, 2, 2, 362, 363, 3, 2, 2, 2, 363, 364, 8, 33, 6, 2, 364, 70, 3, 2, 2, 2, 365, 366, 7, 48, 2, 2, 366, 72, 3, 2, 2, 2, 367, 368, 7, 93, 2, 2, 368, 369, 3, 2, 2, 2, 369, 370, 8, 35, 11, 2, 370, 74, 3, 2, 2, 2, 371, 372, 7, 95, 2, 2, 372, 373, 3, 2, 2, 2, 373, 374, 8, 36, 12, 2, 374, 76, 3, 2, 2, 2, 375, 376, 5, 59, 28, 2, 376, 377, 3, 2, 2, 2, 377, 378, 8, 37, 13, 2, 378, 78, 3, 2, 2, 2, 379, 381, 10, 21, 2, 2, 380, 379, 3, 2, 2, 2, 381, 382, 3, 2, 2, 2, 382, 380, 3, 2, 2, 2, 382, 383, 3, 2, 2, 2, 383, 80, 3, 2, 2, 2, 384, 385, 5, 27, 12, 2, 385, 386, 3, 2, 2, 2, 386, 387, 8, 39, 2, 2, 387, 388, 8, 39, 3, 2, 388, 82, 3, 2, 2, 2, 389, 390, 7, 41, 2, 2, 390, 391, 3, 2, 2, 2, 391, 392, 8, 40, 6, 2, 392, 84, 3, 2, 2, 2, 393, 395, 10, 22, 2, 2, 394, 393, 3, 2, 2, 2, 395, 398, 3, 2, 2, 2, 396, 394, 3, 2, 2, 2, 396, 397, 3, 2, 2, 2, 397, 399, 3, 2, 2, 2, 398, 396, 3, 2, 2, 2, 399, 400, 10, 23, 2, 2, 400, 401, 3, 2, 2, 2, 401, 402, 8, 41, 4, 2, 402, 86, 3, 2, 2, 2, 403, 404, 5, 19, 8, 2, 404, 405, 3, 2, 2, 2, 405, 406, 8, 42, 14, 2, 406, 88, 3, 2, 2, 2, 407, 409, 5, 13, 5, 2, 408, 407, 3, 2, 2, 2, 409, 412, 3, 2, 2, 2, 410, 408, 3, 2, 2, 2, 410, 411, 3, 2, 2, 2, 411, 413, 3, 2, 2, 2, 412, 410, 3, 2, 2, 2, 413, 414, 7, 94, 2, 2, 414, 415, 7, 41, 2, 2, 415, 416, 3, 2, 2, 2, 416, 417, 8, 43, 15, 2, 417, 90, 3, 2, 2, 2, 418, 420, 5, 13, 5, 2, 419, 418, 3, 2, 2, 2, 420, 421, 3, 2, 2, 2, 421, 419, 3, 2, 2, 2, 421, 422, 3, 2, 2, 2, 422, 92, 3, 2, 2, 2, 423, 425, 7, 94, 2, 2, 424, 423, 3, 2, 2, 2, 425, 426, 3, 2, 2, 2, 426, 424, 3, 2, 2, 2, 426, 427, 3, 2, 2, 2, 427, 428, 3, 2, 2, 2, 428, 429, 8, 45, 4, 2, 429, 94, 3, 2, 2, 2, 430, 431, 7, 38, 2, 2, 431, 432, 3, 2, 2, 2, 432, 433, 8, 46, 4, 2, 433, 96, 3, 2, 2, 2, 434, 435, 5, 27, 12, 2, 435, 436, 3, 2, 2, 2, 436, 437, 8, 47, 2, 2, 437, 438, 8, 47, 3, 2, 438, 98, 3, 2, 2, 2, 439, 440, 7, 36, 2, 2, 440, 441, 3, 2, 2, 2, 441, 442, 8, 48, 16, 2, 442, 443, 8, 48, 6, 2, 443, 100, 3, 2, 2, 2, 444, 446, 10, 24, 2, 2, 445, 444, 3, 2, 2, 2, 446, 449, 3, 2, 2, 2, 447, 445, 3, 2, 2, 2, 447, 448, 3, 2, 2, 2, 448, 450, 3, 2, 2, 2, 449, 447, 3, 2, 2, 2, 450, 451, 10, 25, 2, 2, 451, 452, 3, 2, 2, 2, 452, 453, 8, 49, 4, 2, 453, 102, 3, 2, 2, 2, 454, 455, 5, 19, 8, 2, 455, 456, 3, 2, 2, 2, 456, 457, 8, 50, 14, 2, 457, 104, 3, 2, 2, 2, 458, 460, 5, 13, 5, 2, 459, 458, 3, 2, 2, 2, 460, 463, 3, 2, 2, 2, 461, 459, 3, 2, 2, 2, 461, 462, 3, 2, 2, 2, 462, 464, 3, 2, 2, 2, 463, 461, 3, 2, 2, 2, 464, 465, 7, 94, 2, 2, 465, 466, 7, 36, 2, 2, 466, 467, 3, 2, 2, 2, 467, 468, 8, 51, 15, 2, 468, 106, 3, 2, 2, 2, 469, 471, 5, 13, 5, 2, 470, 469, 3, 2, 2, 2, 471, 472, 3, 2, 2, 2, 472, 470, 3, 2, 2, 2, 472, 473, 3, 2, 2, 2, 473, 474, 3, 2, 2, 2, 474, 475, 8, 52, 17, 2, 475, 108, 3, 2, 2, 2, 476, 478, 7, 94, 2, 2, 477, 476, 3, 2, 2, 2, 478, 479, 3, 2, 2, 2, 479, 477, 3, 2, 2, 2, 479, 480, 3, 2, 2, 2, 480, 481, 3, 2, 2, 2, 481, 482, 8, 53, 4, 2, 482, 110, 3, 2, 2, 2, 483, 484, 7, 38, 2, 2, 484, 485, 3, 2, 2, 2, 485, 486, 8, 54, 4, 2, 486, 112, 3, 2, 2, 2, 54, 2, 3, 4, 5, 6, 120, 125, 128, 141, 149, 159, 164, 176, 182, 187, 202, 206, 210, 213, 218, 222, 228, 233, 238, 241, 245, 249, 253, 258, 262, 272, 275, 288, 299, 304, 306, 332, 334, 339, 343, 349, 353, 359, 382, 396, 410, 421, 426, 447, 461, 472, 479, 18, 9, 6, 2, 7, 4, 2, 9, 3, 2, 7, 3, 2, 6, 2, 2, 7, 5, 2, 7, 6, 2, 9, 14, 2, 4, 3, 2, 9, 12, 2, 9, 13, 2, 9, 20, 2, 9, 4, 2, 9, 21, 2, 9, 26, 2, 9, 27, 2] \ No newline at end of file diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.py b/omegaconf/grammar/gen/OmegaConfGrammarLexer.py new file mode 100644 index 000000000..762a09e71 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarLexer.py @@ -0,0 +1,404 @@ +# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarLexer.g4 by ANTLR 4.9.3 +from antlr4 import * +from io import StringIO +import sys + +if sys.version_info[1] > 5: + from typing import TextIO +else: + from typing.io import TextIO + + +def serializedATN(): + with StringIO() as buf: + buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\36") + buf.write("\u01e7\b\1\b\1\b\1\b\1\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5") + buf.write("\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13") + buf.write("\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t") + buf.write("\21\4\22\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26") + buf.write("\4\27\t\27\4\30\t\30\4\31\t\31\4\32\t\32\4\33\t\33\4\34") + buf.write('\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!\t!\4"\t') + buf.write("\"\4#\t#\4$\t$\4%\t%\4&\t&\4'\t'\4(\t(\4)\t)\4*\t*\4") + buf.write("+\t+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62") + buf.write("\t\62\4\63\t\63\4\64\t\64\4\65\t\65\4\66\t\66\3\2\3\2") + buf.write("\3\3\3\3\3\4\3\4\3\4\5\4y\n\4\3\4\7\4|\n\4\f\4\16\4\177") + buf.write("\13\4\5\4\u0081\n\4\3\5\3\5\3\5\3\6\3\6\3\6\3\6\3\6\3") + buf.write("\7\7\7\u008c\n\7\f\7\16\7\u008f\13\7\3\7\3\7\3\b\7\b\u0094") + buf.write("\n\b\f\b\16\b\u0097\13\b\3\b\3\b\3\b\3\b\3\t\6\t\u009e") + buf.write("\n\t\r\t\16\t\u009f\3\n\6\n\u00a3\n\n\r\n\16\n\u00a4\3") + buf.write("\n\3\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\3\f\5\f\u00b1\n") + buf.write("\f\3\f\3\f\3\r\3\r\5\r\u00b7\n\r\3\r\3\r\3\16\5\16\u00bc") + buf.write("\n\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\20\3\20") + buf.write("\3\20\3\20\3\21\5\21\u00cb\n\21\3\21\3\21\5\21\u00cf\n") + buf.write("\21\3\22\3\22\5\22\u00d3\n\22\3\23\5\23\u00d6\n\23\3\23") + buf.write("\3\23\3\24\5\24\u00db\n\24\3\24\3\24\5\24\u00df\n\24\3") + buf.write("\25\3\25\3\25\3\25\5\25\u00e5\n\25\3\25\3\25\3\25\5\25") + buf.write("\u00ea\n\25\3\25\7\25\u00ed\n\25\f\25\16\25\u00f0\13\25") + buf.write("\5\25\u00f2\n\25\3\26\3\26\5\26\u00f6\n\26\3\26\3\26\5") + buf.write("\26\u00fa\n\26\3\26\3\26\5\26\u00fe\n\26\3\26\7\26\u0101") + buf.write("\n\26\f\26\16\26\u0104\13\26\3\27\5\27\u0107\n\27\3\27") + buf.write("\3\27\3\27\3\27\3\27\3\27\3\27\3\27\5\27\u0111\n\27\3") + buf.write("\30\5\30\u0114\n\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31") + buf.write("\3\31\3\31\3\31\3\31\5\31\u0121\n\31\3\32\3\32\3\32\3") + buf.write("\32\3\32\3\33\3\33\3\34\3\34\5\34\u012c\n\34\3\34\3\34") + buf.write("\3\34\7\34\u0131\n\34\f\34\16\34\u0134\13\34\3\35\3\35") + buf.write("\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35") + buf.write("\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\6\35") + buf.write("\u014d\n\35\r\35\16\35\u014e\3\36\6\36\u0152\n\36\r\36") + buf.write("\16\36\u0153\3\37\3\37\5\37\u0158\n\37\3\37\3\37\3\37") + buf.write("\3 \5 \u015e\n \3 \3 \5 \u0162\n \3 \3 \3 \3!\5!\u0168") + buf.write('\n!\3!\3!\3!\3!\3"\3"\3#\3#\3#\3#\3$\3$\3$\3$\3%\3%') + buf.write("\3%\3%\3&\6&\u017d\n&\r&\16&\u017e\3'\3'\3'\3'\3'") + buf.write("\3(\3(\3(\3(\3)\7)\u018b\n)\f)\16)\u018e\13)\3)\3)\3)") + buf.write("\3)\3*\3*\3*\3*\3+\7+\u0199\n+\f+\16+\u019c\13+\3+\3+") + buf.write("\3+\3+\3+\3,\6,\u01a4\n,\r,\16,\u01a5\3-\6-\u01a9\n-\r") + buf.write("-\16-\u01aa\3-\3-\3.\3.\3.\3.\3/\3/\3/\3/\3/\3\60\3\60") + buf.write("\3\60\3\60\3\60\3\61\7\61\u01be\n\61\f\61\16\61\u01c1") + buf.write("\13\61\3\61\3\61\3\61\3\61\3\62\3\62\3\62\3\62\3\63\7") + buf.write("\63\u01cc\n\63\f\63\16\63\u01cf\13\63\3\63\3\63\3\63\3") + buf.write("\63\3\63\3\64\6\64\u01d7\n\64\r\64\16\64\u01d8\3\64\3") + buf.write("\64\3\65\6\65\u01de\n\65\r\65\16\65\u01df\3\65\3\65\3") + buf.write("\66\3\66\3\66\3\66\2\2\67\7\2\t\2\13\2\r\2\17\2\21\3\23") + buf.write("\4\25\5\27\2\31\34\33\6\35\7\37\b!\t#\n%\13'\f)\r+\16") + buf.write("-\2/\2\61\17\63\20\65\21\67\229\23;\24=\25?\26A\2C\2E") + buf.write("\27G\30I\35K\36M\2O\31Q\2S\32U\2W\2Y\2[\33]\2_\2a\2c\2") + buf.write("e\2g\2i\2k\2m\2o\2\7\2\3\4\5\6\32\4\2C\\c|\3\2\62;\3\2") + buf.write("\63;\3\2&&\4\2&&^^\4\2GGgg\4\2--//\4\2KKkk\4\2PPpp\4\2") + buf.write("HHhh\4\2CCcc\4\2VVvv\4\2TTtt\4\2WWww\4\2NNnn\4\2UUuu\b") + buf.write('\2&\',-/\61AB^^~~\4\2//aa\4\2\13\13""\13\2\13\13""') + buf.write("$$)+\60\60<<]_}}\177\177\4\2&&))\5\2&&))^^\4\2$$&&\5\2") + buf.write("$$&&^^\2\u0218\2\17\3\2\2\2\2\21\3\2\2\2\2\23\3\2\2\2") + buf.write("\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3\2\2\2\3\33\3\2\2\2\3") + buf.write("\35\3\2\2\2\3\37\3\2\2\2\3!\3\2\2\2\3#\3\2\2\2\3%\3\2") + buf.write("\2\2\3'\3\2\2\2\3)\3\2\2\2\3+\3\2\2\2\3\61\3\2\2\2\3") + buf.write("\63\3\2\2\2\3\65\3\2\2\2\3\67\3\2\2\2\39\3\2\2\2\3;\3") + buf.write("\2\2\2\3=\3\2\2\2\3?\3\2\2\2\4A\3\2\2\2\4C\3\2\2\2\4E") + buf.write("\3\2\2\2\4G\3\2\2\2\4I\3\2\2\2\4K\3\2\2\2\4M\3\2\2\2\4") + buf.write("O\3\2\2\2\5Q\3\2\2\2\5S\3\2\2\2\5U\3\2\2\2\5W\3\2\2\2") + buf.write("\5Y\3\2\2\2\5[\3\2\2\2\5]\3\2\2\2\5_\3\2\2\2\6a\3\2\2") + buf.write("\2\6c\3\2\2\2\6e\3\2\2\2\6g\3\2\2\2\6i\3\2\2\2\6k\3\2") + buf.write("\2\2\6m\3\2\2\2\6o\3\2\2\2\7q\3\2\2\2\ts\3\2\2\2\13\u0080") + buf.write("\3\2\2\2\r\u0082\3\2\2\2\17\u0085\3\2\2\2\21\u008d\3\2") + buf.write("\2\2\23\u0095\3\2\2\2\25\u009d\3\2\2\2\27\u00a2\3\2\2") + buf.write("\2\31\u00a8\3\2\2\2\33\u00ac\3\2\2\2\35\u00b4\3\2\2\2") + buf.write("\37\u00bb\3\2\2\2!\u00c1\3\2\2\2#\u00c5\3\2\2\2%\u00ca") + buf.write("\3\2\2\2'\u00d0\3\2\2\2)\u00d5\3\2\2\2+\u00da\3\2\2\2") + buf.write("-\u00f1\3\2\2\2/\u00f5\3\2\2\2\61\u0106\3\2\2\2\63\u0113") + buf.write("\3\2\2\2\65\u0120\3\2\2\2\67\u0122\3\2\2\29\u0127\3\2") + buf.write("\2\2;\u012b\3\2\2\2=\u014c\3\2\2\2?\u0151\3\2\2\2A\u0155") + buf.write("\3\2\2\2C\u015d\3\2\2\2E\u0167\3\2\2\2G\u016d\3\2\2\2") + buf.write("I\u016f\3\2\2\2K\u0173\3\2\2\2M\u0177\3\2\2\2O\u017c\3") + buf.write("\2\2\2Q\u0180\3\2\2\2S\u0185\3\2\2\2U\u018c\3\2\2\2W\u0193") + buf.write("\3\2\2\2Y\u019a\3\2\2\2[\u01a3\3\2\2\2]\u01a8\3\2\2\2") + buf.write("_\u01ae\3\2\2\2a\u01b2\3\2\2\2c\u01b7\3\2\2\2e\u01bf\3") + buf.write("\2\2\2g\u01c6\3\2\2\2i\u01cd\3\2\2\2k\u01d6\3\2\2\2m\u01dd") + buf.write("\3\2\2\2o\u01e3\3\2\2\2qr\t\2\2\2r\b\3\2\2\2st\t\3\2\2") + buf.write("t\n\3\2\2\2u\u0081\7\62\2\2v}\t\4\2\2wy\7a\2\2xw\3\2\2") + buf.write("\2xy\3\2\2\2yz\3\2\2\2z|\5\t\3\2{x\3\2\2\2|\177\3\2\2") + buf.write("\2}{\3\2\2\2}~\3\2\2\2~\u0081\3\2\2\2\177}\3\2\2\2\u0080") + buf.write("u\3\2\2\2\u0080v\3\2\2\2\u0081\f\3\2\2\2\u0082\u0083\7") + buf.write("^\2\2\u0083\u0084\7^\2\2\u0084\16\3\2\2\2\u0085\u0086") + buf.write("\5\33\f\2\u0086\u0087\3\2\2\2\u0087\u0088\b\6\2\2\u0088") + buf.write("\u0089\b\6\3\2\u0089\20\3\2\2\2\u008a\u008c\n\5\2\2\u008b") + buf.write("\u008a\3\2\2\2\u008c\u008f\3\2\2\2\u008d\u008b\3\2\2\2") + buf.write("\u008d\u008e\3\2\2\2\u008e\u0090\3\2\2\2\u008f\u008d\3") + buf.write("\2\2\2\u0090\u0091\n\6\2\2\u0091\22\3\2\2\2\u0092\u0094") + buf.write("\5\r\5\2\u0093\u0092\3\2\2\2\u0094\u0097\3\2\2\2\u0095") + buf.write("\u0093\3\2\2\2\u0095\u0096\3\2\2\2\u0096\u0098\3\2\2\2") + buf.write("\u0097\u0095\3\2\2\2\u0098\u0099\7^\2\2\u0099\u009a\7") + buf.write("&\2\2\u009a\u009b\7}\2\2\u009b\24\3\2\2\2\u009c\u009e") + buf.write("\5\r\5\2\u009d\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f") + buf.write("\u009d\3\2\2\2\u009f\u00a0\3\2\2\2\u00a0\26\3\2\2\2\u00a1") + buf.write("\u00a3\7^\2\2\u00a2\u00a1\3\2\2\2\u00a3\u00a4\3\2\2\2") + buf.write("\u00a4\u00a2\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5\u00a6\3") + buf.write("\2\2\2\u00a6\u00a7\b\n\4\2\u00a7\30\3\2\2\2\u00a8\u00a9") + buf.write("\7&\2\2\u00a9\u00aa\3\2\2\2\u00aa\u00ab\b\13\4\2\u00ab") + buf.write("\32\3\2\2\2\u00ac\u00ad\7&\2\2\u00ad\u00ae\7}\2\2\u00ae") + buf.write("\u00b0\3\2\2\2\u00af\u00b1\5?\36\2\u00b0\u00af\3\2\2\2") + buf.write("\u00b0\u00b1\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b3\b") + buf.write("\f\3\2\u00b3\34\3\2\2\2\u00b4\u00b6\7}\2\2\u00b5\u00b7") + buf.write("\5?\36\2\u00b6\u00b5\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7") + buf.write("\u00b8\3\2\2\2\u00b8\u00b9\b\r\5\2\u00b9\36\3\2\2\2\u00ba") + buf.write("\u00bc\5?\36\2\u00bb\u00ba\3\2\2\2\u00bb\u00bc\3\2\2\2") + buf.write("\u00bc\u00bd\3\2\2\2\u00bd\u00be\7\177\2\2\u00be\u00bf") + buf.write("\3\2\2\2\u00bf\u00c0\b\16\6\2\u00c0 \3\2\2\2\u00c1\u00c2") + buf.write("\7)\2\2\u00c2\u00c3\3\2\2\2\u00c3\u00c4\b\17\7\2\u00c4") + buf.write('"\3\2\2\2\u00c5\u00c6\7$\2\2\u00c6\u00c7\3\2\2\2\u00c7') + buf.write("\u00c8\b\20\b\2\u00c8$\3\2\2\2\u00c9\u00cb\5?\36\2\u00ca") + buf.write("\u00c9\3\2\2\2\u00ca\u00cb\3\2\2\2\u00cb\u00cc\3\2\2\2") + buf.write("\u00cc\u00ce\7.\2\2\u00cd\u00cf\5?\36\2\u00ce\u00cd\3") + buf.write("\2\2\2\u00ce\u00cf\3\2\2\2\u00cf&\3\2\2\2\u00d0\u00d2") + buf.write("\7]\2\2\u00d1\u00d3\5?\36\2\u00d2\u00d1\3\2\2\2\u00d2") + buf.write("\u00d3\3\2\2\2\u00d3(\3\2\2\2\u00d4\u00d6\5?\36\2\u00d5") + buf.write("\u00d4\3\2\2\2\u00d5\u00d6\3\2\2\2\u00d6\u00d7\3\2\2\2") + buf.write("\u00d7\u00d8\7_\2\2\u00d8*\3\2\2\2\u00d9\u00db\5?\36\2") + buf.write("\u00da\u00d9\3\2\2\2\u00da\u00db\3\2\2\2\u00db\u00dc\3") + buf.write("\2\2\2\u00dc\u00de\7<\2\2\u00dd\u00df\5?\36\2\u00de\u00dd") + buf.write("\3\2\2\2\u00de\u00df\3\2\2\2\u00df,\3\2\2\2\u00e0\u00e1") + buf.write("\5\13\4\2\u00e1\u00e2\7\60\2\2\u00e2\u00f2\3\2\2\2\u00e3") + buf.write("\u00e5\5\13\4\2\u00e4\u00e3\3\2\2\2\u00e4\u00e5\3\2\2") + buf.write("\2\u00e5\u00e6\3\2\2\2\u00e6\u00e7\7\60\2\2\u00e7\u00ee") + buf.write("\5\t\3\2\u00e8\u00ea\7a\2\2\u00e9\u00e8\3\2\2\2\u00e9") + buf.write("\u00ea\3\2\2\2\u00ea\u00eb\3\2\2\2\u00eb\u00ed\5\t\3\2") + buf.write("\u00ec\u00e9\3\2\2\2\u00ed\u00f0\3\2\2\2\u00ee\u00ec\3") + buf.write("\2\2\2\u00ee\u00ef\3\2\2\2\u00ef\u00f2\3\2\2\2\u00f0\u00ee") + buf.write("\3\2\2\2\u00f1\u00e0\3\2\2\2\u00f1\u00e4\3\2\2\2\u00f2") + buf.write(".\3\2\2\2\u00f3\u00f6\5\13\4\2\u00f4\u00f6\5-\25\2\u00f5") + buf.write("\u00f3\3\2\2\2\u00f5\u00f4\3\2\2\2\u00f6\u00f7\3\2\2\2") + buf.write("\u00f7\u00f9\t\7\2\2\u00f8\u00fa\t\b\2\2\u00f9\u00f8\3") + buf.write("\2\2\2\u00f9\u00fa\3\2\2\2\u00fa\u00fb\3\2\2\2\u00fb\u0102") + buf.write("\5\t\3\2\u00fc\u00fe\7a\2\2\u00fd\u00fc\3\2\2\2\u00fd") + buf.write("\u00fe\3\2\2\2\u00fe\u00ff\3\2\2\2\u00ff\u0101\5\t\3\2") + buf.write("\u0100\u00fd\3\2\2\2\u0101\u0104\3\2\2\2\u0102\u0100\3") + buf.write("\2\2\2\u0102\u0103\3\2\2\2\u0103\60\3\2\2\2\u0104\u0102") + buf.write("\3\2\2\2\u0105\u0107\t\b\2\2\u0106\u0105\3\2\2\2\u0106") + buf.write("\u0107\3\2\2\2\u0107\u0110\3\2\2\2\u0108\u0111\5-\25\2") + buf.write("\u0109\u0111\5/\26\2\u010a\u010b\t\t\2\2\u010b\u010c\t") + buf.write("\n\2\2\u010c\u0111\t\13\2\2\u010d\u010e\t\n\2\2\u010e") + buf.write("\u010f\t\f\2\2\u010f\u0111\t\n\2\2\u0110\u0108\3\2\2\2") + buf.write("\u0110\u0109\3\2\2\2\u0110\u010a\3\2\2\2\u0110\u010d\3") + buf.write("\2\2\2\u0111\62\3\2\2\2\u0112\u0114\t\b\2\2\u0113\u0112") + buf.write("\3\2\2\2\u0113\u0114\3\2\2\2\u0114\u0115\3\2\2\2\u0115") + buf.write("\u0116\5\13\4\2\u0116\64\3\2\2\2\u0117\u0118\t\r\2\2\u0118") + buf.write("\u0119\t\16\2\2\u0119\u011a\t\17\2\2\u011a\u0121\t\7\2") + buf.write("\2\u011b\u011c\t\13\2\2\u011c\u011d\t\f\2\2\u011d\u011e") + buf.write("\t\20\2\2\u011e\u011f\t\21\2\2\u011f\u0121\t\7\2\2\u0120") + buf.write("\u0117\3\2\2\2\u0120\u011b\3\2\2\2\u0121\66\3\2\2\2\u0122") + buf.write("\u0123\t\n\2\2\u0123\u0124\t\17\2\2\u0124\u0125\t\20\2") + buf.write("\2\u0125\u0126\t\20\2\2\u01268\3\2\2\2\u0127\u0128\t\22") + buf.write("\2\2\u0128:\3\2\2\2\u0129\u012c\5\7\2\2\u012a\u012c\7") + buf.write("a\2\2\u012b\u0129\3\2\2\2\u012b\u012a\3\2\2\2\u012c\u0132") + buf.write("\3\2\2\2\u012d\u0131\5\7\2\2\u012e\u0131\5\t\3\2\u012f") + buf.write("\u0131\t\23\2\2\u0130\u012d\3\2\2\2\u0130\u012e\3\2\2") + buf.write("\2\u0130\u012f\3\2\2\2\u0131\u0134\3\2\2\2\u0132\u0130") + buf.write("\3\2\2\2\u0132\u0133\3\2\2\2\u0133<\3\2\2\2\u0134\u0132") + buf.write("\3\2\2\2\u0135\u014d\5\r\5\2\u0136\u0137\7^\2\2\u0137") + buf.write("\u014d\7*\2\2\u0138\u0139\7^\2\2\u0139\u014d\7+\2\2\u013a") + buf.write("\u013b\7^\2\2\u013b\u014d\7]\2\2\u013c\u013d\7^\2\2\u013d") + buf.write("\u014d\7_\2\2\u013e\u013f\7^\2\2\u013f\u014d\7}\2\2\u0140") + buf.write("\u0141\7^\2\2\u0141\u014d\7\177\2\2\u0142\u0143\7^\2\2") + buf.write("\u0143\u014d\7<\2\2\u0144\u0145\7^\2\2\u0145\u014d\7?") + buf.write("\2\2\u0146\u0147\7^\2\2\u0147\u014d\7.\2\2\u0148\u0149") + buf.write('\7^\2\2\u0149\u014d\7"\2\2\u014a\u014b\7^\2\2\u014b\u014d') + buf.write("\7\13\2\2\u014c\u0135\3\2\2\2\u014c\u0136\3\2\2\2\u014c") + buf.write("\u0138\3\2\2\2\u014c\u013a\3\2\2\2\u014c\u013c\3\2\2\2") + buf.write("\u014c\u013e\3\2\2\2\u014c\u0140\3\2\2\2\u014c\u0142\3") + buf.write("\2\2\2\u014c\u0144\3\2\2\2\u014c\u0146\3\2\2\2\u014c\u0148") + buf.write("\3\2\2\2\u014c\u014a\3\2\2\2\u014d\u014e\3\2\2\2\u014e") + buf.write("\u014c\3\2\2\2\u014e\u014f\3\2\2\2\u014f>\3\2\2\2\u0150") + buf.write("\u0152\t\24\2\2\u0151\u0150\3\2\2\2\u0152\u0153\3\2\2") + buf.write("\2\u0153\u0151\3\2\2\2\u0153\u0154\3\2\2\2\u0154@\3\2") + buf.write("\2\2\u0155\u0157\5\33\f\2\u0156\u0158\5?\36\2\u0157\u0156") + buf.write("\3\2\2\2\u0157\u0158\3\2\2\2\u0158\u0159\3\2\2\2\u0159") + buf.write("\u015a\b\37\2\2\u015a\u015b\b\37\3\2\u015bB\3\2\2\2\u015c") + buf.write("\u015e\5?\36\2\u015d\u015c\3\2\2\2\u015d\u015e\3\2\2\2") + buf.write("\u015e\u015f\3\2\2\2\u015f\u0161\7<\2\2\u0160\u0162\5") + buf.write("?\36\2\u0161\u0160\3\2\2\2\u0161\u0162\3\2\2\2\u0162\u0163") + buf.write("\3\2\2\2\u0163\u0164\b \t\2\u0164\u0165\b \n\2\u0165D") + buf.write("\3\2\2\2\u0166\u0168\5?\36\2\u0167\u0166\3\2\2\2\u0167") + buf.write("\u0168\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u016a\7\177\2") + buf.write("\2\u016a\u016b\3\2\2\2\u016b\u016c\b!\6\2\u016cF\3\2\2") + buf.write("\2\u016d\u016e\7\60\2\2\u016eH\3\2\2\2\u016f\u0170\7]") + buf.write("\2\2\u0170\u0171\3\2\2\2\u0171\u0172\b#\13\2\u0172J\3") + buf.write("\2\2\2\u0173\u0174\7_\2\2\u0174\u0175\3\2\2\2\u0175\u0176") + buf.write("\b$\f\2\u0176L\3\2\2\2\u0177\u0178\5;\34\2\u0178\u0179") + buf.write("\3\2\2\2\u0179\u017a\b%\r\2\u017aN\3\2\2\2\u017b\u017d") + buf.write("\n\25\2\2\u017c\u017b\3\2\2\2\u017d\u017e\3\2\2\2\u017e") + buf.write("\u017c\3\2\2\2\u017e\u017f\3\2\2\2\u017fP\3\2\2\2\u0180") + buf.write("\u0181\5\33\f\2\u0181\u0182\3\2\2\2\u0182\u0183\b'\2") + buf.write("\2\u0183\u0184\b'\3\2\u0184R\3\2\2\2\u0185\u0186\7)\2") + buf.write("\2\u0186\u0187\3\2\2\2\u0187\u0188\b(\6\2\u0188T\3\2\2") + buf.write("\2\u0189\u018b\n\26\2\2\u018a\u0189\3\2\2\2\u018b\u018e") + buf.write("\3\2\2\2\u018c\u018a\3\2\2\2\u018c\u018d\3\2\2\2\u018d") + buf.write("\u018f\3\2\2\2\u018e\u018c\3\2\2\2\u018f\u0190\n\27\2") + buf.write("\2\u0190\u0191\3\2\2\2\u0191\u0192\b)\4\2\u0192V\3\2\2") + buf.write("\2\u0193\u0194\5\23\b\2\u0194\u0195\3\2\2\2\u0195\u0196") + buf.write("\b*\16\2\u0196X\3\2\2\2\u0197\u0199\5\r\5\2\u0198\u0197") + buf.write("\3\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2\2\2\u019a") + buf.write("\u019b\3\2\2\2\u019b\u019d\3\2\2\2\u019c\u019a\3\2\2\2") + buf.write("\u019d\u019e\7^\2\2\u019e\u019f\7)\2\2\u019f\u01a0\3\2") + buf.write("\2\2\u01a0\u01a1\b+\17\2\u01a1Z\3\2\2\2\u01a2\u01a4\5") + buf.write("\r\5\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3") + buf.write("\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\\\3\2\2\2\u01a7\u01a9") + buf.write("\7^\2\2\u01a8\u01a7\3\2\2\2\u01a9\u01aa\3\2\2\2\u01aa") + buf.write("\u01a8\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2\2\2") + buf.write("\u01ac\u01ad\b-\4\2\u01ad^\3\2\2\2\u01ae\u01af\7&\2\2") + buf.write("\u01af\u01b0\3\2\2\2\u01b0\u01b1\b.\4\2\u01b1`\3\2\2\2") + buf.write("\u01b2\u01b3\5\33\f\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5") + buf.write("\b/\2\2\u01b5\u01b6\b/\3\2\u01b6b\3\2\2\2\u01b7\u01b8") + buf.write("\7$\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01ba\b\60\20\2\u01ba") + buf.write("\u01bb\b\60\6\2\u01bbd\3\2\2\2\u01bc\u01be\n\30\2\2\u01bd") + buf.write("\u01bc\3\2\2\2\u01be\u01c1\3\2\2\2\u01bf\u01bd\3\2\2\2") + buf.write("\u01bf\u01c0\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1\u01bf\3") + buf.write("\2\2\2\u01c2\u01c3\n\31\2\2\u01c3\u01c4\3\2\2\2\u01c4") + buf.write("\u01c5\b\61\4\2\u01c5f\3\2\2\2\u01c6\u01c7\5\23\b\2\u01c7") + buf.write("\u01c8\3\2\2\2\u01c8\u01c9\b\62\16\2\u01c9h\3\2\2\2\u01ca") + buf.write("\u01cc\5\r\5\2\u01cb\u01ca\3\2\2\2\u01cc\u01cf\3\2\2\2") + buf.write("\u01cd\u01cb\3\2\2\2\u01cd\u01ce\3\2\2\2\u01ce\u01d0\3") + buf.write("\2\2\2\u01cf\u01cd\3\2\2\2\u01d0\u01d1\7^\2\2\u01d1\u01d2") + buf.write("\7$\2\2\u01d2\u01d3\3\2\2\2\u01d3\u01d4\b\63\17\2\u01d4") + buf.write("j\3\2\2\2\u01d5\u01d7\5\r\5\2\u01d6\u01d5\3\2\2\2\u01d7") + buf.write("\u01d8\3\2\2\2\u01d8\u01d6\3\2\2\2\u01d8\u01d9\3\2\2\2") + buf.write("\u01d9\u01da\3\2\2\2\u01da\u01db\b\64\21\2\u01dbl\3\2") + buf.write("\2\2\u01dc\u01de\7^\2\2\u01dd\u01dc\3\2\2\2\u01de\u01df") + buf.write("\3\2\2\2\u01df\u01dd\3\2\2\2\u01df\u01e0\3\2\2\2\u01e0") + buf.write("\u01e1\3\2\2\2\u01e1\u01e2\b\65\4\2\u01e2n\3\2\2\2\u01e3") + buf.write("\u01e4\7&\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e6\b\66\4\2") + buf.write("\u01e6p\3\2\2\2\66\2\3\4\5\6x}\u0080\u008d\u0095\u009f") + buf.write("\u00a4\u00b0\u00b6\u00bb\u00ca\u00ce\u00d2\u00d5\u00da") + buf.write("\u00de\u00e4\u00e9\u00ee\u00f1\u00f5\u00f9\u00fd\u0102") + buf.write("\u0106\u0110\u0113\u0120\u012b\u0130\u0132\u014c\u014e") + buf.write("\u0153\u0157\u015d\u0161\u0167\u017e\u018c\u019a\u01a5") + buf.write("\u01aa\u01bf\u01cd\u01d8\u01df\22\t\6\2\7\4\2\t\3\2\7") + buf.write("\3\2\6\2\2\7\5\2\7\6\2\t\16\2\4\3\2\t\f\2\t\r\2\t\24\2") + buf.write("\t\4\2\t\25\2\t\32\2\t\33\2") + return buf.getvalue() + + +class OmegaConfGrammarLexer(Lexer): + atn = ATNDeserializer().deserialize(serializedATN()) + + decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)] + + VALUE_MODE = 1 + INTERPOLATION_MODE = 2 + QUOTED_SINGLE_MODE = 3 + QUOTED_DOUBLE_MODE = 4 + + ANY_STR = 1 + ESC_INTER = 2 + TOP_ESC = 3 + INTER_OPEN = 4 + BRACE_OPEN = 5 + BRACE_CLOSE = 6 + QUOTE_OPEN_SINGLE = 7 + QUOTE_OPEN_DOUBLE = 8 + COMMA = 9 + BRACKET_OPEN = 10 + BRACKET_CLOSE = 11 + COLON = 12 + FLOAT = 13 + INT = 14 + BOOL = 15 + NULL = 16 + UNQUOTED_CHAR = 17 + ID = 18 + ESC = 19 + WS = 20 + INTER_CLOSE = 21 + DOT = 22 + INTER_KEY = 23 + MATCHING_QUOTE_CLOSE = 24 + QUOTED_ESC = 25 + DOLLAR = 26 + INTER_BRACKET_OPEN = 27 + INTER_BRACKET_CLOSE = 28 + + channelNames = ["DEFAULT_TOKEN_CHANNEL", "HIDDEN"] + + modeNames = [ + "DEFAULT_MODE", + "VALUE_MODE", + "INTERPOLATION_MODE", + "QUOTED_SINGLE_MODE", + "QUOTED_DOUBLE_MODE", + ] + + literalNames = ["", "'.'", "'['", "']'"] + + symbolicNames = [ + "", + "ANY_STR", + "ESC_INTER", + "TOP_ESC", + "INTER_OPEN", + "BRACE_OPEN", + "BRACE_CLOSE", + "QUOTE_OPEN_SINGLE", + "QUOTE_OPEN_DOUBLE", + "COMMA", + "BRACKET_OPEN", + "BRACKET_CLOSE", + "COLON", + "FLOAT", + "INT", + "BOOL", + "NULL", + "UNQUOTED_CHAR", + "ID", + "ESC", + "WS", + "INTER_CLOSE", + "DOT", + "INTER_KEY", + "MATCHING_QUOTE_CLOSE", + "QUOTED_ESC", + "DOLLAR", + "INTER_BRACKET_OPEN", + "INTER_BRACKET_CLOSE", + ] + + ruleNames = [ + "CHAR", + "DIGIT", + "INT_UNSIGNED", + "ESC_BACKSLASH", + "TOP_INTER_OPEN", + "ANY_STR", + "ESC_INTER", + "TOP_ESC", + "BACKSLASHES", + "DOLLAR", + "INTER_OPEN", + "BRACE_OPEN", + "BRACE_CLOSE", + "QUOTE_OPEN_SINGLE", + "QUOTE_OPEN_DOUBLE", + "COMMA", + "BRACKET_OPEN", + "BRACKET_CLOSE", + "COLON", + "POINT_FLOAT", + "EXPONENT_FLOAT", + "FLOAT", + "INT", + "BOOL", + "NULL", + "UNQUOTED_CHAR", + "ID", + "ESC", + "WS", + "NESTED_INTER_OPEN", + "INTER_COLON", + "INTER_CLOSE", + "DOT", + "INTER_BRACKET_OPEN", + "INTER_BRACKET_CLOSE", + "INTER_ID", + "INTER_KEY", + "QSINGLE_INTER_OPEN", + "MATCHING_QUOTE_CLOSE", + "QSINGLE_STR", + "QSINGLE_ESC_INTER", + "QSINGLE_ESC_QUOTE", + "QUOTED_ESC", + "QSINGLE_BACKSLASHES", + "QSINGLE_DOLLAR", + "QDOUBLE_INTER_OPEN", + "QDOUBLE_CLOSE", + "QDOUBLE_STR", + "QDOUBLE_ESC_INTER", + "QDOUBLE_ESC_QUOTE", + "QDOUBLE_ESC", + "QDOUBLE_BACKSLASHES", + "QDOUBLE_DOLLAR", + ] + + grammarFileName = "OmegaConfGrammarLexer.g4" + + def __init__(self, input=None, output: TextIO = sys.stdout): + super().__init__(input, output) + self.checkVersion("4.9.3") + self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) + self._actions = None + self._predicates = None diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens b/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens new file mode 100644 index 000000000..c22963b90 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens @@ -0,0 +1,31 @@ +ANY_STR=1 +ESC_INTER=2 +TOP_ESC=3 +INTER_OPEN=4 +BRACE_OPEN=5 +BRACE_CLOSE=6 +QUOTE_OPEN_SINGLE=7 +QUOTE_OPEN_DOUBLE=8 +COMMA=9 +BRACKET_OPEN=10 +BRACKET_CLOSE=11 +COLON=12 +FLOAT=13 +INT=14 +BOOL=15 +NULL=16 +UNQUOTED_CHAR=17 +ID=18 +ESC=19 +WS=20 +INTER_CLOSE=21 +DOT=22 +INTER_KEY=23 +MATCHING_QUOTE_CLOSE=24 +QUOTED_ESC=25 +DOLLAR=26 +INTER_BRACKET_OPEN=27 +INTER_BRACKET_CLOSE=28 +'.'=22 +'['=27 +']'=28 diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.interp b/omegaconf/grammar/gen/OmegaConfGrammarParser.interp new file mode 100644 index 000000000..b99c77c0a --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarParser.interp @@ -0,0 +1,83 @@ +token literal names: +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +null +'.' +null +null +null +null +'[' +']' + +token symbolic names: +null +ANY_STR +ESC_INTER +TOP_ESC +INTER_OPEN +BRACE_OPEN +BRACE_CLOSE +QUOTE_OPEN_SINGLE +QUOTE_OPEN_DOUBLE +COMMA +BRACKET_OPEN +BRACKET_CLOSE +COLON +FLOAT +INT +BOOL +NULL +UNQUOTED_CHAR +ID +ESC +WS +INTER_CLOSE +DOT +INTER_KEY +MATCHING_QUOTE_CLOSE +QUOTED_ESC +DOLLAR +INTER_BRACKET_OPEN +INTER_BRACKET_CLOSE + +rule names: +configValue +singleElement +text +element +listContainer +dictContainer +dictKeyValuePair +sequence +interpolation +interpolationNode +interpolationResolver +configKey +resolverName +quotedValue +primitive +dictKey + + +atn: +[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 3, 30, 183, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 6, 4, 47, 10, 4, 13, 4, 14, 4, 48, 3, 5, 3, 5, 3, 5, 3, 5, 5, 5, 55, 10, 5, 3, 6, 3, 6, 5, 6, 59, 10, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 67, 10, 7, 12, 7, 14, 7, 70, 11, 7, 5, 7, 72, 10, 7, 3, 7, 3, 7, 3, 8, 3, 8, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 5, 9, 83, 10, 9, 7, 9, 85, 10, 9, 12, 9, 14, 9, 88, 11, 9, 3, 9, 3, 9, 5, 9, 92, 10, 9, 6, 9, 94, 10, 9, 13, 9, 14, 9, 95, 5, 9, 98, 10, 9, 3, 10, 3, 10, 5, 10, 102, 10, 10, 3, 11, 3, 11, 7, 11, 106, 10, 11, 12, 11, 14, 11, 109, 11, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 5, 11, 116, 10, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 7, 11, 124, 10, 11, 12, 11, 14, 11, 127, 11, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 5, 12, 135, 10, 12, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, 5, 13, 142, 10, 13, 3, 14, 3, 14, 5, 14, 146, 10, 14, 3, 14, 3, 14, 3, 14, 5, 14, 151, 10, 14, 7, 14, 153, 10, 14, 12, 14, 14, 14, 156, 11, 14, 3, 15, 3, 15, 5, 15, 160, 10, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 6, 16, 174, 10, 16, 13, 16, 14, 16, 175, 3, 17, 6, 17, 179, 10, 17, 13, 17, 14, 17, 180, 3, 17, 2, 2, 18, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 2, 4, 3, 2, 9, 10, 3, 2, 15, 22, 2, 206, 2, 34, 3, 2, 2, 2, 4, 37, 3, 2, 2, 2, 6, 46, 3, 2, 2, 2, 8, 54, 3, 2, 2, 2, 10, 56, 3, 2, 2, 2, 12, 62, 3, 2, 2, 2, 14, 75, 3, 2, 2, 2, 16, 97, 3, 2, 2, 2, 18, 101, 3, 2, 2, 2, 20, 103, 3, 2, 2, 2, 22, 130, 3, 2, 2, 2, 24, 141, 3, 2, 2, 2, 26, 145, 3, 2, 2, 2, 28, 157, 3, 2, 2, 2, 30, 173, 3, 2, 2, 2, 32, 178, 3, 2, 2, 2, 34, 35, 5, 6, 4, 2, 35, 36, 7, 2, 2, 3, 36, 3, 3, 2, 2, 2, 37, 38, 5, 8, 5, 2, 38, 39, 7, 2, 2, 3, 39, 5, 3, 2, 2, 2, 40, 47, 5, 18, 10, 2, 41, 47, 7, 3, 2, 2, 42, 47, 7, 21, 2, 2, 43, 47, 7, 4, 2, 2, 44, 47, 7, 5, 2, 2, 45, 47, 7, 27, 2, 2, 46, 40, 3, 2, 2, 2, 46, 41, 3, 2, 2, 2, 46, 42, 3, 2, 2, 2, 46, 43, 3, 2, 2, 2, 46, 44, 3, 2, 2, 2, 46, 45, 3, 2, 2, 2, 47, 48, 3, 2, 2, 2, 48, 46, 3, 2, 2, 2, 48, 49, 3, 2, 2, 2, 49, 7, 3, 2, 2, 2, 50, 55, 5, 30, 16, 2, 51, 55, 5, 28, 15, 2, 52, 55, 5, 10, 6, 2, 53, 55, 5, 12, 7, 2, 54, 50, 3, 2, 2, 2, 54, 51, 3, 2, 2, 2, 54, 52, 3, 2, 2, 2, 54, 53, 3, 2, 2, 2, 55, 9, 3, 2, 2, 2, 56, 58, 7, 12, 2, 2, 57, 59, 5, 16, 9, 2, 58, 57, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 60, 3, 2, 2, 2, 60, 61, 7, 13, 2, 2, 61, 11, 3, 2, 2, 2, 62, 71, 7, 7, 2, 2, 63, 68, 5, 14, 8, 2, 64, 65, 7, 11, 2, 2, 65, 67, 5, 14, 8, 2, 66, 64, 3, 2, 2, 2, 67, 70, 3, 2, 2, 2, 68, 66, 3, 2, 2, 2, 68, 69, 3, 2, 2, 2, 69, 72, 3, 2, 2, 2, 70, 68, 3, 2, 2, 2, 71, 63, 3, 2, 2, 2, 71, 72, 3, 2, 2, 2, 72, 73, 3, 2, 2, 2, 73, 74, 7, 8, 2, 2, 74, 13, 3, 2, 2, 2, 75, 76, 5, 32, 17, 2, 76, 77, 7, 14, 2, 2, 77, 78, 5, 8, 5, 2, 78, 15, 3, 2, 2, 2, 79, 86, 5, 8, 5, 2, 80, 82, 7, 11, 2, 2, 81, 83, 5, 8, 5, 2, 82, 81, 3, 2, 2, 2, 82, 83, 3, 2, 2, 2, 83, 85, 3, 2, 2, 2, 84, 80, 3, 2, 2, 2, 85, 88, 3, 2, 2, 2, 86, 84, 3, 2, 2, 2, 86, 87, 3, 2, 2, 2, 87, 98, 3, 2, 2, 2, 88, 86, 3, 2, 2, 2, 89, 91, 7, 11, 2, 2, 90, 92, 5, 8, 5, 2, 91, 90, 3, 2, 2, 2, 91, 92, 3, 2, 2, 2, 92, 94, 3, 2, 2, 2, 93, 89, 3, 2, 2, 2, 94, 95, 3, 2, 2, 2, 95, 93, 3, 2, 2, 2, 95, 96, 3, 2, 2, 2, 96, 98, 3, 2, 2, 2, 97, 79, 3, 2, 2, 2, 97, 93, 3, 2, 2, 2, 98, 17, 3, 2, 2, 2, 99, 102, 5, 20, 11, 2, 100, 102, 5, 22, 12, 2, 101, 99, 3, 2, 2, 2, 101, 100, 3, 2, 2, 2, 102, 19, 3, 2, 2, 2, 103, 107, 7, 6, 2, 2, 104, 106, 7, 24, 2, 2, 105, 104, 3, 2, 2, 2, 106, 109, 3, 2, 2, 2, 107, 105, 3, 2, 2, 2, 107, 108, 3, 2, 2, 2, 108, 115, 3, 2, 2, 2, 109, 107, 3, 2, 2, 2, 110, 116, 5, 24, 13, 2, 111, 112, 7, 12, 2, 2, 112, 113, 5, 24, 13, 2, 113, 114, 7, 13, 2, 2, 114, 116, 3, 2, 2, 2, 115, 110, 3, 2, 2, 2, 115, 111, 3, 2, 2, 2, 116, 125, 3, 2, 2, 2, 117, 118, 7, 24, 2, 2, 118, 124, 5, 24, 13, 2, 119, 120, 7, 12, 2, 2, 120, 121, 5, 24, 13, 2, 121, 122, 7, 13, 2, 2, 122, 124, 3, 2, 2, 2, 123, 117, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 127, 3, 2, 2, 2, 125, 123, 3, 2, 2, 2, 125, 126, 3, 2, 2, 2, 126, 128, 3, 2, 2, 2, 127, 125, 3, 2, 2, 2, 128, 129, 7, 23, 2, 2, 129, 21, 3, 2, 2, 2, 130, 131, 7, 6, 2, 2, 131, 132, 5, 26, 14, 2, 132, 134, 7, 14, 2, 2, 133, 135, 5, 16, 9, 2, 134, 133, 3, 2, 2, 2, 134, 135, 3, 2, 2, 2, 135, 136, 3, 2, 2, 2, 136, 137, 7, 8, 2, 2, 137, 23, 3, 2, 2, 2, 138, 142, 5, 18, 10, 2, 139, 142, 7, 20, 2, 2, 140, 142, 7, 25, 2, 2, 141, 138, 3, 2, 2, 2, 141, 139, 3, 2, 2, 2, 141, 140, 3, 2, 2, 2, 142, 25, 3, 2, 2, 2, 143, 146, 5, 18, 10, 2, 144, 146, 7, 20, 2, 2, 145, 143, 3, 2, 2, 2, 145, 144, 3, 2, 2, 2, 146, 154, 3, 2, 2, 2, 147, 150, 7, 24, 2, 2, 148, 151, 5, 18, 10, 2, 149, 151, 7, 20, 2, 2, 150, 148, 3, 2, 2, 2, 150, 149, 3, 2, 2, 2, 151, 153, 3, 2, 2, 2, 152, 147, 3, 2, 2, 2, 153, 156, 3, 2, 2, 2, 154, 152, 3, 2, 2, 2, 154, 155, 3, 2, 2, 2, 155, 27, 3, 2, 2, 2, 156, 154, 3, 2, 2, 2, 157, 159, 9, 2, 2, 2, 158, 160, 5, 6, 4, 2, 159, 158, 3, 2, 2, 2, 159, 160, 3, 2, 2, 2, 160, 161, 3, 2, 2, 2, 161, 162, 7, 26, 2, 2, 162, 29, 3, 2, 2, 2, 163, 174, 7, 20, 2, 2, 164, 174, 7, 18, 2, 2, 165, 174, 7, 16, 2, 2, 166, 174, 7, 15, 2, 2, 167, 174, 7, 17, 2, 2, 168, 174, 7, 19, 2, 2, 169, 174, 7, 14, 2, 2, 170, 174, 7, 21, 2, 2, 171, 174, 7, 22, 2, 2, 172, 174, 5, 18, 10, 2, 173, 163, 3, 2, 2, 2, 173, 164, 3, 2, 2, 2, 173, 165, 3, 2, 2, 2, 173, 166, 3, 2, 2, 2, 173, 167, 3, 2, 2, 2, 173, 168, 3, 2, 2, 2, 173, 169, 3, 2, 2, 2, 173, 170, 3, 2, 2, 2, 173, 171, 3, 2, 2, 2, 173, 172, 3, 2, 2, 2, 174, 175, 3, 2, 2, 2, 175, 173, 3, 2, 2, 2, 175, 176, 3, 2, 2, 2, 176, 31, 3, 2, 2, 2, 177, 179, 9, 3, 2, 2, 178, 177, 3, 2, 2, 2, 179, 180, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 180, 181, 3, 2, 2, 2, 181, 33, 3, 2, 2, 2, 27, 46, 48, 54, 58, 68, 71, 82, 86, 91, 95, 97, 101, 107, 115, 123, 125, 134, 141, 145, 150, 154, 159, 173, 175, 180] \ No newline at end of file diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.py b/omegaconf/grammar/gen/OmegaConfGrammarParser.py new file mode 100644 index 000000000..81cde7563 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarParser.py @@ -0,0 +1,1745 @@ +# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 +# encoding: utf-8 +from antlr4 import * +from io import StringIO +import sys + +if sys.version_info[1] > 5: + from typing import TextIO +else: + from typing.io import TextIO + + +def serializedATN(): + with StringIO() as buf: + buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3\36") + buf.write("\u00b7\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7") + buf.write("\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r\4\16") + buf.write("\t\16\4\17\t\17\4\20\t\20\4\21\t\21\3\2\3\2\3\2\3\3\3") + buf.write("\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\6\4/\n\4\r\4\16\4\60\3") + buf.write("\5\3\5\3\5\3\5\5\5\67\n\5\3\6\3\6\5\6;\n\6\3\6\3\6\3\7") + buf.write("\3\7\3\7\3\7\7\7C\n\7\f\7\16\7F\13\7\5\7H\n\7\3\7\3\7") + buf.write("\3\b\3\b\3\b\3\b\3\t\3\t\3\t\5\tS\n\t\7\tU\n\t\f\t\16") + buf.write("\tX\13\t\3\t\3\t\5\t\\\n\t\6\t^\n\t\r\t\16\t_\5\tb\n\t") + buf.write("\3\n\3\n\5\nf\n\n\3\13\3\13\7\13j\n\13\f\13\16\13m\13") + buf.write("\13\3\13\3\13\3\13\3\13\3\13\5\13t\n\13\3\13\3\13\3\13") + buf.write("\3\13\3\13\3\13\7\13|\n\13\f\13\16\13\177\13\13\3\13\3") + buf.write("\13\3\f\3\f\3\f\3\f\5\f\u0087\n\f\3\f\3\f\3\r\3\r\3\r") + buf.write("\5\r\u008e\n\r\3\16\3\16\5\16\u0092\n\16\3\16\3\16\3\16") + buf.write("\5\16\u0097\n\16\7\16\u0099\n\16\f\16\16\16\u009c\13\16") + buf.write("\3\17\3\17\5\17\u00a0\n\17\3\17\3\17\3\20\3\20\3\20\3") + buf.write("\20\3\20\3\20\3\20\3\20\3\20\3\20\6\20\u00ae\n\20\r\20") + buf.write("\16\20\u00af\3\21\6\21\u00b3\n\21\r\21\16\21\u00b4\3\21") + buf.write("\2\2\22\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \2\4\3") + buf.write('\2\t\n\3\2\17\26\2\u00ce\2"\3\2\2\2\4%\3\2\2\2\6.\3\2') + buf.write("\2\2\b\66\3\2\2\2\n8\3\2\2\2\f>\3\2\2\2\16K\3\2\2\2\20") + buf.write("a\3\2\2\2\22e\3\2\2\2\24g\3\2\2\2\26\u0082\3\2\2\2\30") + buf.write("\u008d\3\2\2\2\32\u0091\3\2\2\2\34\u009d\3\2\2\2\36\u00ad") + buf.write('\3\2\2\2 \u00b2\3\2\2\2"#\5\6\4\2#$\7\2\2\3$\3\3\2\2') + buf.write("\2%&\5\b\5\2&'\7\2\2\3'\5\3\2\2\2(/\5\22\n\2)/\7\3\2") + buf.write("\2*/\7\25\2\2+/\7\4\2\2,/\7\5\2\2-/\7\33\2\2.(\3\2\2\2") + buf.write(".)\3\2\2\2.*\3\2\2\2.+\3\2\2\2.,\3\2\2\2.-\3\2\2\2/\60") + buf.write("\3\2\2\2\60.\3\2\2\2\60\61\3\2\2\2\61\7\3\2\2\2\62\67") + buf.write("\5\36\20\2\63\67\5\34\17\2\64\67\5\n\6\2\65\67\5\f\7\2") + buf.write("\66\62\3\2\2\2\66\63\3\2\2\2\66\64\3\2\2\2\66\65\3\2\2") + buf.write("\2\67\t\3\2\2\28:\7\f\2\29;\5\20\t\2:9\3\2\2\2:;\3\2\2") + buf.write("\2;<\3\2\2\2<=\7\r\2\2=\13\3\2\2\2>G\7\7\2\2?D\5\16\b") + buf.write("\2@A\7\13\2\2AC\5\16\b\2B@\3\2\2\2CF\3\2\2\2DB\3\2\2\2") + buf.write("DE\3\2\2\2EH\3\2\2\2FD\3\2\2\2G?\3\2\2\2GH\3\2\2\2HI\3") + buf.write("\2\2\2IJ\7\b\2\2J\r\3\2\2\2KL\5 \21\2LM\7\16\2\2MN\5\b") + buf.write("\5\2N\17\3\2\2\2OV\5\b\5\2PR\7\13\2\2QS\5\b\5\2RQ\3\2") + buf.write("\2\2RS\3\2\2\2SU\3\2\2\2TP\3\2\2\2UX\3\2\2\2VT\3\2\2\2") + buf.write("VW\3\2\2\2Wb\3\2\2\2XV\3\2\2\2Y[\7\13\2\2Z\\\5\b\5\2[") + buf.write("Z\3\2\2\2[\\\3\2\2\2\\^\3\2\2\2]Y\3\2\2\2^_\3\2\2\2_]") + buf.write("\3\2\2\2_`\3\2\2\2`b\3\2\2\2aO\3\2\2\2a]\3\2\2\2b\21\3") + buf.write("\2\2\2cf\5\24\13\2df\5\26\f\2ec\3\2\2\2ed\3\2\2\2f\23") + buf.write("\3\2\2\2gk\7\6\2\2hj\7\30\2\2ih\3\2\2\2jm\3\2\2\2ki\3") + buf.write("\2\2\2kl\3\2\2\2ls\3\2\2\2mk\3\2\2\2nt\5\30\r\2op\7\f") + buf.write("\2\2pq\5\30\r\2qr\7\r\2\2rt\3\2\2\2sn\3\2\2\2so\3\2\2") + buf.write("\2t}\3\2\2\2uv\7\30\2\2v|\5\30\r\2wx\7\f\2\2xy\5\30\r") + buf.write("\2yz\7\r\2\2z|\3\2\2\2{u\3\2\2\2{w\3\2\2\2|\177\3\2\2") + buf.write("\2}{\3\2\2\2}~\3\2\2\2~\u0080\3\2\2\2\177}\3\2\2\2\u0080") + buf.write("\u0081\7\27\2\2\u0081\25\3\2\2\2\u0082\u0083\7\6\2\2\u0083") + buf.write("\u0084\5\32\16\2\u0084\u0086\7\16\2\2\u0085\u0087\5\20") + buf.write("\t\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088") + buf.write("\3\2\2\2\u0088\u0089\7\b\2\2\u0089\27\3\2\2\2\u008a\u008e") + buf.write("\5\22\n\2\u008b\u008e\7\24\2\2\u008c\u008e\7\31\2\2\u008d") + buf.write("\u008a\3\2\2\2\u008d\u008b\3\2\2\2\u008d\u008c\3\2\2\2") + buf.write("\u008e\31\3\2\2\2\u008f\u0092\5\22\n\2\u0090\u0092\7\24") + buf.write("\2\2\u0091\u008f\3\2\2\2\u0091\u0090\3\2\2\2\u0092\u009a") + buf.write("\3\2\2\2\u0093\u0096\7\30\2\2\u0094\u0097\5\22\n\2\u0095") + buf.write("\u0097\7\24\2\2\u0096\u0094\3\2\2\2\u0096\u0095\3\2\2") + buf.write("\2\u0097\u0099\3\2\2\2\u0098\u0093\3\2\2\2\u0099\u009c") + buf.write("\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2\2\2\u009b") + buf.write("\33\3\2\2\2\u009c\u009a\3\2\2\2\u009d\u009f\t\2\2\2\u009e") + buf.write("\u00a0\5\6\4\2\u009f\u009e\3\2\2\2\u009f\u00a0\3\2\2\2") + buf.write("\u00a0\u00a1\3\2\2\2\u00a1\u00a2\7\32\2\2\u00a2\35\3\2") + buf.write("\2\2\u00a3\u00ae\7\24\2\2\u00a4\u00ae\7\22\2\2\u00a5\u00ae") + buf.write("\7\20\2\2\u00a6\u00ae\7\17\2\2\u00a7\u00ae\7\21\2\2\u00a8") + buf.write("\u00ae\7\23\2\2\u00a9\u00ae\7\16\2\2\u00aa\u00ae\7\25") + buf.write("\2\2\u00ab\u00ae\7\26\2\2\u00ac\u00ae\5\22\n\2\u00ad\u00a3") + buf.write("\3\2\2\2\u00ad\u00a4\3\2\2\2\u00ad\u00a5\3\2\2\2\u00ad") + buf.write("\u00a6\3\2\2\2\u00ad\u00a7\3\2\2\2\u00ad\u00a8\3\2\2\2") + buf.write("\u00ad\u00a9\3\2\2\2\u00ad\u00aa\3\2\2\2\u00ad\u00ab\3") + buf.write("\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00ad") + buf.write("\3\2\2\2\u00af\u00b0\3\2\2\2\u00b0\37\3\2\2\2\u00b1\u00b3") + buf.write("\t\3\2\2\u00b2\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4") + buf.write("\u00b2\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5!\3\2\2\2\33.") + buf.write("\60\66:DGRV[_aeks{}\u0086\u008d\u0091\u0096\u009a\u009f") + buf.write("\u00ad\u00af\u00b4") + return buf.getvalue() + + +class OmegaConfGrammarParser(Parser): + grammarFileName = "OmegaConfGrammarParser.g4" + + atn = ATNDeserializer().deserialize(serializedATN()) + + decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)] + + sharedContextCache = PredictionContextCache() + + literalNames = [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "'.'", + "", + "", + "", + "", + "'['", + "']'", + ] + + symbolicNames = [ + "", + "ANY_STR", + "ESC_INTER", + "TOP_ESC", + "INTER_OPEN", + "BRACE_OPEN", + "BRACE_CLOSE", + "QUOTE_OPEN_SINGLE", + "QUOTE_OPEN_DOUBLE", + "COMMA", + "BRACKET_OPEN", + "BRACKET_CLOSE", + "COLON", + "FLOAT", + "INT", + "BOOL", + "NULL", + "UNQUOTED_CHAR", + "ID", + "ESC", + "WS", + "INTER_CLOSE", + "DOT", + "INTER_KEY", + "MATCHING_QUOTE_CLOSE", + "QUOTED_ESC", + "DOLLAR", + "INTER_BRACKET_OPEN", + "INTER_BRACKET_CLOSE", + ] + + RULE_configValue = 0 + RULE_singleElement = 1 + RULE_text = 2 + RULE_element = 3 + RULE_listContainer = 4 + RULE_dictContainer = 5 + RULE_dictKeyValuePair = 6 + RULE_sequence = 7 + RULE_interpolation = 8 + RULE_interpolationNode = 9 + RULE_interpolationResolver = 10 + RULE_configKey = 11 + RULE_resolverName = 12 + RULE_quotedValue = 13 + RULE_primitive = 14 + RULE_dictKey = 15 + + ruleNames = [ + "configValue", + "singleElement", + "text", + "element", + "listContainer", + "dictContainer", + "dictKeyValuePair", + "sequence", + "interpolation", + "interpolationNode", + "interpolationResolver", + "configKey", + "resolverName", + "quotedValue", + "primitive", + "dictKey", + ] + + EOF = Token.EOF + ANY_STR = 1 + ESC_INTER = 2 + TOP_ESC = 3 + INTER_OPEN = 4 + BRACE_OPEN = 5 + BRACE_CLOSE = 6 + QUOTE_OPEN_SINGLE = 7 + QUOTE_OPEN_DOUBLE = 8 + COMMA = 9 + BRACKET_OPEN = 10 + BRACKET_CLOSE = 11 + COLON = 12 + FLOAT = 13 + INT = 14 + BOOL = 15 + NULL = 16 + UNQUOTED_CHAR = 17 + ID = 18 + ESC = 19 + WS = 20 + INTER_CLOSE = 21 + DOT = 22 + INTER_KEY = 23 + MATCHING_QUOTE_CLOSE = 24 + QUOTED_ESC = 25 + DOLLAR = 26 + INTER_BRACKET_OPEN = 27 + INTER_BRACKET_CLOSE = 28 + + def __init__(self, input: TokenStream, output: TextIO = sys.stdout): + super().__init__(input, output) + self.checkVersion("4.9.3") + self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) + self._predicates = None + + class ConfigValueContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def text(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.TextContext, 0) + + def EOF(self): + return self.getToken(OmegaConfGrammarParser.EOF, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_configValue + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterConfigValue"): + listener.enterConfigValue(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitConfigValue"): + listener.exitConfigValue(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitConfigValue"): + return visitor.visitConfigValue(self) + else: + return visitor.visitChildren(self) + + def configValue(self): + localctx = OmegaConfGrammarParser.ConfigValueContext(self, self._ctx, self.state) + self.enterRule(localctx, 0, self.RULE_configValue) + try: + self.enterOuterAlt(localctx, 1) + self.state = 32 + self.text() + self.state = 33 + self.match(OmegaConfGrammarParser.EOF) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class SingleElementContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def element(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, 0) + + def EOF(self): + return self.getToken(OmegaConfGrammarParser.EOF, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_singleElement + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterSingleElement"): + listener.enterSingleElement(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitSingleElement"): + listener.exitSingleElement(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitSingleElement"): + return visitor.visitSingleElement(self) + else: + return visitor.visitChildren(self) + + def singleElement(self): + localctx = OmegaConfGrammarParser.SingleElementContext(self, self._ctx, self.state) + self.enterRule(localctx, 2, self.RULE_singleElement) + try: + self.enterOuterAlt(localctx, 1) + self.state = 35 + self.element() + self.state = 36 + self.match(OmegaConfGrammarParser.EOF) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class TextContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def interpolation(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) + + def ANY_STR(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ANY_STR) + else: + return self.getToken(OmegaConfGrammarParser.ANY_STR, i) + + def ESC(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ESC) + else: + return self.getToken(OmegaConfGrammarParser.ESC, i) + + def ESC_INTER(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ESC_INTER) + else: + return self.getToken(OmegaConfGrammarParser.ESC_INTER, i) + + def TOP_ESC(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.TOP_ESC) + else: + return self.getToken(OmegaConfGrammarParser.TOP_ESC, i) + + def QUOTED_ESC(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.QUOTED_ESC) + else: + return self.getToken(OmegaConfGrammarParser.QUOTED_ESC, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_text + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterText"): + listener.enterText(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitText"): + listener.exitText(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitText"): + return visitor.visitText(self) + else: + return visitor.visitChildren(self) + + def text(self): + localctx = OmegaConfGrammarParser.TextContext(self, self._ctx, self.state) + self.enterRule(localctx, 4, self.RULE_text) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 44 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 44 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.INTER_OPEN]: + self.state = 38 + self.interpolation() + pass + elif token in [OmegaConfGrammarParser.ANY_STR]: + self.state = 39 + self.match(OmegaConfGrammarParser.ANY_STR) + pass + elif token in [OmegaConfGrammarParser.ESC]: + self.state = 40 + self.match(OmegaConfGrammarParser.ESC) + pass + elif token in [OmegaConfGrammarParser.ESC_INTER]: + self.state = 41 + self.match(OmegaConfGrammarParser.ESC_INTER) + pass + elif token in [OmegaConfGrammarParser.TOP_ESC]: + self.state = 42 + self.match(OmegaConfGrammarParser.TOP_ESC) + pass + elif token in [OmegaConfGrammarParser.QUOTED_ESC]: + self.state = 43 + self.match(OmegaConfGrammarParser.QUOTED_ESC) + pass + else: + raise NoViableAltException(self) + + self.state = 46 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not ( + ( + ((_la) & ~0x3F) == 0 + and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.ANY_STR) + | (1 << OmegaConfGrammarParser.ESC_INTER) + | (1 << OmegaConfGrammarParser.TOP_ESC) + | (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.QUOTED_ESC) + ) + ) + != 0 + ) + ): + break + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ElementContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def primitive(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.PrimitiveContext, 0) + + def quotedValue(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.QuotedValueContext, 0) + + def listContainer(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.ListContainerContext, 0) + + def dictContainer(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.DictContainerContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_element + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterElement"): + listener.enterElement(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitElement"): + listener.exitElement(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitElement"): + return visitor.visitElement(self) + else: + return visitor.visitChildren(self) + + def element(self): + localctx = OmegaConfGrammarParser.ElementContext(self, self._ctx, self.state) + self.enterRule(localctx, 6, self.RULE_element) + try: + self.state = 52 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [ + OmegaConfGrammarParser.INTER_OPEN, + OmegaConfGrammarParser.COLON, + OmegaConfGrammarParser.FLOAT, + OmegaConfGrammarParser.INT, + OmegaConfGrammarParser.BOOL, + OmegaConfGrammarParser.NULL, + OmegaConfGrammarParser.UNQUOTED_CHAR, + OmegaConfGrammarParser.ID, + OmegaConfGrammarParser.ESC, + OmegaConfGrammarParser.WS, + ]: + self.enterOuterAlt(localctx, 1) + self.state = 48 + self.primitive() + pass + elif token in [ + OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, + OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, + ]: + self.enterOuterAlt(localctx, 2) + self.state = 49 + self.quotedValue() + pass + elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: + self.enterOuterAlt(localctx, 3) + self.state = 50 + self.listContainer() + pass + elif token in [OmegaConfGrammarParser.BRACE_OPEN]: + self.enterOuterAlt(localctx, 4) + self.state = 51 + self.dictContainer() + pass + else: + raise NoViableAltException(self) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ListContainerContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def BRACKET_OPEN(self): + return self.getToken(OmegaConfGrammarParser.BRACKET_OPEN, 0) + + def BRACKET_CLOSE(self): + return self.getToken(OmegaConfGrammarParser.BRACKET_CLOSE, 0) + + def sequence(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.SequenceContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_listContainer + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterListContainer"): + listener.enterListContainer(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitListContainer"): + listener.exitListContainer(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitListContainer"): + return visitor.visitListContainer(self) + else: + return visitor.visitChildren(self) + + def listContainer(self): + localctx = OmegaConfGrammarParser.ListContainerContext(self, self._ctx, self.state) + self.enterRule(localctx, 8, self.RULE_listContainer) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 54 + self.match(OmegaConfGrammarParser.BRACKET_OPEN) + self.state = 56 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.BRACE_OPEN) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) + | (1 << OmegaConfGrammarParser.COMMA) + | (1 << OmegaConfGrammarParser.BRACKET_OPEN) + | (1 << OmegaConfGrammarParser.COLON) + | (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) != 0: + self.state = 55 + self.sequence() + + self.state = 58 + self.match(OmegaConfGrammarParser.BRACKET_CLOSE) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class DictContainerContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def BRACE_OPEN(self): + return self.getToken(OmegaConfGrammarParser.BRACE_OPEN, 0) + + def BRACE_CLOSE(self): + return self.getToken(OmegaConfGrammarParser.BRACE_CLOSE, 0) + + def dictKeyValuePair(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.DictKeyValuePairContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.DictKeyValuePairContext, i) + + def COMMA(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.COMMA) + else: + return self.getToken(OmegaConfGrammarParser.COMMA, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_dictContainer + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterDictContainer"): + listener.enterDictContainer(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitDictContainer"): + listener.exitDictContainer(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitDictContainer"): + return visitor.visitDictContainer(self) + else: + return visitor.visitChildren(self) + + def dictContainer(self): + localctx = OmegaConfGrammarParser.DictContainerContext(self, self._ctx, self.state) + self.enterRule(localctx, 10, self.RULE_dictContainer) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 60 + self.match(OmegaConfGrammarParser.BRACE_OPEN) + self.state = 69 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) != 0: + self.state = 61 + self.dictKeyValuePair() + self.state = 66 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la == OmegaConfGrammarParser.COMMA: + self.state = 62 + self.match(OmegaConfGrammarParser.COMMA) + self.state = 63 + self.dictKeyValuePair() + self.state = 68 + self._errHandler.sync(self) + _la = self._input.LA(1) + + self.state = 71 + self.match(OmegaConfGrammarParser.BRACE_CLOSE) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class DictKeyValuePairContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def dictKey(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.DictKeyContext, 0) + + def COLON(self): + return self.getToken(OmegaConfGrammarParser.COLON, 0) + + def element(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_dictKeyValuePair + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterDictKeyValuePair"): + listener.enterDictKeyValuePair(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitDictKeyValuePair"): + listener.exitDictKeyValuePair(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitDictKeyValuePair"): + return visitor.visitDictKeyValuePair(self) + else: + return visitor.visitChildren(self) + + def dictKeyValuePair(self): + localctx = OmegaConfGrammarParser.DictKeyValuePairContext(self, self._ctx, self.state) + self.enterRule(localctx, 12, self.RULE_dictKeyValuePair) + try: + self.enterOuterAlt(localctx, 1) + self.state = 73 + self.dictKey() + self.state = 74 + self.match(OmegaConfGrammarParser.COLON) + self.state = 75 + self.element() + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class SequenceContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def element(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.ElementContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, i) + + def COMMA(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.COMMA) + else: + return self.getToken(OmegaConfGrammarParser.COMMA, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_sequence + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterSequence"): + listener.enterSequence(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitSequence"): + listener.exitSequence(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitSequence"): + return visitor.visitSequence(self) + else: + return visitor.visitChildren(self) + + def sequence(self): + localctx = OmegaConfGrammarParser.SequenceContext(self, self._ctx, self.state) + self.enterRule(localctx, 14, self.RULE_sequence) + self._la = 0 # Token type + try: + self.state = 95 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [ + OmegaConfGrammarParser.INTER_OPEN, + OmegaConfGrammarParser.BRACE_OPEN, + OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, + OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, + OmegaConfGrammarParser.BRACKET_OPEN, + OmegaConfGrammarParser.COLON, + OmegaConfGrammarParser.FLOAT, + OmegaConfGrammarParser.INT, + OmegaConfGrammarParser.BOOL, + OmegaConfGrammarParser.NULL, + OmegaConfGrammarParser.UNQUOTED_CHAR, + OmegaConfGrammarParser.ID, + OmegaConfGrammarParser.ESC, + OmegaConfGrammarParser.WS, + ]: + self.enterOuterAlt(localctx, 1) + self.state = 77 + self.element() + self.state = 84 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la == OmegaConfGrammarParser.COMMA: + self.state = 78 + self.match(OmegaConfGrammarParser.COMMA) + self.state = 80 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.BRACE_OPEN) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) + | (1 << OmegaConfGrammarParser.BRACKET_OPEN) + | (1 << OmegaConfGrammarParser.COLON) + | (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) != 0: + self.state = 79 + self.element() + + self.state = 86 + self._errHandler.sync(self) + _la = self._input.LA(1) + + pass + elif token in [OmegaConfGrammarParser.COMMA]: + self.enterOuterAlt(localctx, 2) + self.state = 91 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 87 + self.match(OmegaConfGrammarParser.COMMA) + self.state = 89 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.BRACE_OPEN) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) + | (1 << OmegaConfGrammarParser.BRACKET_OPEN) + | (1 << OmegaConfGrammarParser.COLON) + | (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) != 0: + self.state = 88 + self.element() + + self.state = 93 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not (_la == OmegaConfGrammarParser.COMMA): + break + + pass + else: + raise NoViableAltException(self) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class InterpolationContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def interpolationNode(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationNodeContext, 0) + + def interpolationResolver(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationResolverContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_interpolation + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterInterpolation"): + listener.enterInterpolation(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitInterpolation"): + listener.exitInterpolation(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitInterpolation"): + return visitor.visitInterpolation(self) + else: + return visitor.visitChildren(self) + + def interpolation(self): + localctx = OmegaConfGrammarParser.InterpolationContext(self, self._ctx, self.state) + self.enterRule(localctx, 16, self.RULE_interpolation) + try: + self.state = 99 + self._errHandler.sync(self) + la_ = self._interp.adaptivePredict(self._input, 11, self._ctx) + if la_ == 1: + self.enterOuterAlt(localctx, 1) + self.state = 97 + self.interpolationNode() + pass + + elif la_ == 2: + self.enterOuterAlt(localctx, 2) + self.state = 98 + self.interpolationResolver() + pass + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class InterpolationNodeContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def INTER_OPEN(self): + return self.getToken(OmegaConfGrammarParser.INTER_OPEN, 0) + + def INTER_CLOSE(self): + return self.getToken(OmegaConfGrammarParser.INTER_CLOSE, 0) + + def configKey(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.ConfigKeyContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.ConfigKeyContext, i) + + def BRACKET_OPEN(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.BRACKET_OPEN) + else: + return self.getToken(OmegaConfGrammarParser.BRACKET_OPEN, i) + + def BRACKET_CLOSE(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.BRACKET_CLOSE) + else: + return self.getToken(OmegaConfGrammarParser.BRACKET_CLOSE, i) + + def DOT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.DOT) + else: + return self.getToken(OmegaConfGrammarParser.DOT, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_interpolationNode + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterInterpolationNode"): + listener.enterInterpolationNode(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitInterpolationNode"): + listener.exitInterpolationNode(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitInterpolationNode"): + return visitor.visitInterpolationNode(self) + else: + return visitor.visitChildren(self) + + def interpolationNode(self): + localctx = OmegaConfGrammarParser.InterpolationNodeContext(self, self._ctx, self.state) + self.enterRule(localctx, 18, self.RULE_interpolationNode) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 101 + self.match(OmegaConfGrammarParser.INTER_OPEN) + self.state = 105 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la == OmegaConfGrammarParser.DOT: + self.state = 102 + self.match(OmegaConfGrammarParser.DOT) + self.state = 107 + self._errHandler.sync(self) + _la = self._input.LA(1) + + self.state = 113 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [ + OmegaConfGrammarParser.INTER_OPEN, + OmegaConfGrammarParser.ID, + OmegaConfGrammarParser.INTER_KEY, + ]: + self.state = 108 + self.configKey() + pass + elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: + self.state = 109 + self.match(OmegaConfGrammarParser.BRACKET_OPEN) + self.state = 110 + self.configKey() + self.state = 111 + self.match(OmegaConfGrammarParser.BRACKET_CLOSE) + pass + else: + raise NoViableAltException(self) + + self.state = 123 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la == OmegaConfGrammarParser.BRACKET_OPEN or _la == OmegaConfGrammarParser.DOT: + self.state = 121 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.DOT]: + self.state = 115 + self.match(OmegaConfGrammarParser.DOT) + self.state = 116 + self.configKey() + pass + elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: + self.state = 117 + self.match(OmegaConfGrammarParser.BRACKET_OPEN) + self.state = 118 + self.configKey() + self.state = 119 + self.match(OmegaConfGrammarParser.BRACKET_CLOSE) + pass + else: + raise NoViableAltException(self) + + self.state = 125 + self._errHandler.sync(self) + _la = self._input.LA(1) + + self.state = 126 + self.match(OmegaConfGrammarParser.INTER_CLOSE) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class InterpolationResolverContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def INTER_OPEN(self): + return self.getToken(OmegaConfGrammarParser.INTER_OPEN, 0) + + def resolverName(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.ResolverNameContext, 0) + + def COLON(self): + return self.getToken(OmegaConfGrammarParser.COLON, 0) + + def BRACE_CLOSE(self): + return self.getToken(OmegaConfGrammarParser.BRACE_CLOSE, 0) + + def sequence(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.SequenceContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_interpolationResolver + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterInterpolationResolver"): + listener.enterInterpolationResolver(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitInterpolationResolver"): + listener.exitInterpolationResolver(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitInterpolationResolver"): + return visitor.visitInterpolationResolver(self) + else: + return visitor.visitChildren(self) + + def interpolationResolver(self): + localctx = OmegaConfGrammarParser.InterpolationResolverContext(self, self._ctx, self.state) + self.enterRule(localctx, 20, self.RULE_interpolationResolver) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 128 + self.match(OmegaConfGrammarParser.INTER_OPEN) + self.state = 129 + self.resolverName() + self.state = 130 + self.match(OmegaConfGrammarParser.COLON) + self.state = 132 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.BRACE_OPEN) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) + | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) + | (1 << OmegaConfGrammarParser.COMMA) + | (1 << OmegaConfGrammarParser.BRACKET_OPEN) + | (1 << OmegaConfGrammarParser.COLON) + | (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) != 0: + self.state = 131 + self.sequence() + + self.state = 134 + self.match(OmegaConfGrammarParser.BRACE_CLOSE) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ConfigKeyContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def interpolation(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, 0) + + def ID(self): + return self.getToken(OmegaConfGrammarParser.ID, 0) + + def INTER_KEY(self): + return self.getToken(OmegaConfGrammarParser.INTER_KEY, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_configKey + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterConfigKey"): + listener.enterConfigKey(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitConfigKey"): + listener.exitConfigKey(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitConfigKey"): + return visitor.visitConfigKey(self) + else: + return visitor.visitChildren(self) + + def configKey(self): + localctx = OmegaConfGrammarParser.ConfigKeyContext(self, self._ctx, self.state) + self.enterRule(localctx, 22, self.RULE_configKey) + try: + self.state = 139 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.INTER_OPEN]: + self.enterOuterAlt(localctx, 1) + self.state = 136 + self.interpolation() + pass + elif token in [OmegaConfGrammarParser.ID]: + self.enterOuterAlt(localctx, 2) + self.state = 137 + self.match(OmegaConfGrammarParser.ID) + pass + elif token in [OmegaConfGrammarParser.INTER_KEY]: + self.enterOuterAlt(localctx, 3) + self.state = 138 + self.match(OmegaConfGrammarParser.INTER_KEY) + pass + else: + raise NoViableAltException(self) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class ResolverNameContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def interpolation(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) + + def ID(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ID) + else: + return self.getToken(OmegaConfGrammarParser.ID, i) + + def DOT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.DOT) + else: + return self.getToken(OmegaConfGrammarParser.DOT, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_resolverName + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterResolverName"): + listener.enterResolverName(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitResolverName"): + listener.exitResolverName(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitResolverName"): + return visitor.visitResolverName(self) + else: + return visitor.visitChildren(self) + + def resolverName(self): + localctx = OmegaConfGrammarParser.ResolverNameContext(self, self._ctx, self.state) + self.enterRule(localctx, 24, self.RULE_resolverName) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 143 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.INTER_OPEN]: + self.state = 141 + self.interpolation() + pass + elif token in [OmegaConfGrammarParser.ID]: + self.state = 142 + self.match(OmegaConfGrammarParser.ID) + pass + else: + raise NoViableAltException(self) + + self.state = 152 + self._errHandler.sync(self) + _la = self._input.LA(1) + while _la == OmegaConfGrammarParser.DOT: + self.state = 145 + self.match(OmegaConfGrammarParser.DOT) + self.state = 148 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.INTER_OPEN]: + self.state = 146 + self.interpolation() + pass + elif token in [OmegaConfGrammarParser.ID]: + self.state = 147 + self.match(OmegaConfGrammarParser.ID) + pass + else: + raise NoViableAltException(self) + + self.state = 154 + self._errHandler.sync(self) + _la = self._input.LA(1) + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class QuotedValueContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def MATCHING_QUOTE_CLOSE(self): + return self.getToken(OmegaConfGrammarParser.MATCHING_QUOTE_CLOSE, 0) + + def QUOTE_OPEN_SINGLE(self): + return self.getToken(OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, 0) + + def QUOTE_OPEN_DOUBLE(self): + return self.getToken(OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, 0) + + def text(self): + return self.getTypedRuleContext(OmegaConfGrammarParser.TextContext, 0) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_quotedValue + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterQuotedValue"): + listener.enterQuotedValue(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitQuotedValue"): + listener.exitQuotedValue(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitQuotedValue"): + return visitor.visitQuotedValue(self) + else: + return visitor.visitChildren(self) + + def quotedValue(self): + localctx = OmegaConfGrammarParser.QuotedValueContext(self, self._ctx, self.state) + self.enterRule(localctx, 26, self.RULE_quotedValue) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 155 + _la = self._input.LA(1) + if not ( + _la == OmegaConfGrammarParser.QUOTE_OPEN_SINGLE + or _la == OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE + ): + self._errHandler.recoverInline(self) + else: + self._errHandler.reportMatch(self) + self.consume() + self.state = 157 + self._errHandler.sync(self) + _la = self._input.LA(1) + if ((_la) & ~0x3F) == 0 and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.ANY_STR) + | (1 << OmegaConfGrammarParser.ESC_INTER) + | (1 << OmegaConfGrammarParser.TOP_ESC) + | (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.QUOTED_ESC) + ) + ) != 0: + self.state = 156 + self.text() + + self.state = 159 + self.match(OmegaConfGrammarParser.MATCHING_QUOTE_CLOSE) + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class PrimitiveContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def ID(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ID) + else: + return self.getToken(OmegaConfGrammarParser.ID, i) + + def NULL(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.NULL) + else: + return self.getToken(OmegaConfGrammarParser.NULL, i) + + def INT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.INT) + else: + return self.getToken(OmegaConfGrammarParser.INT, i) + + def FLOAT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.FLOAT) + else: + return self.getToken(OmegaConfGrammarParser.FLOAT, i) + + def BOOL(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.BOOL) + else: + return self.getToken(OmegaConfGrammarParser.BOOL, i) + + def UNQUOTED_CHAR(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.UNQUOTED_CHAR) + else: + return self.getToken(OmegaConfGrammarParser.UNQUOTED_CHAR, i) + + def COLON(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.COLON) + else: + return self.getToken(OmegaConfGrammarParser.COLON, i) + + def ESC(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ESC) + else: + return self.getToken(OmegaConfGrammarParser.ESC, i) + + def WS(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.WS) + else: + return self.getToken(OmegaConfGrammarParser.WS, i) + + def interpolation(self, i: int = None): + if i is None: + return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) + else: + return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_primitive + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterPrimitive"): + listener.enterPrimitive(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitPrimitive"): + listener.exitPrimitive(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitPrimitive"): + return visitor.visitPrimitive(self) + else: + return visitor.visitChildren(self) + + def primitive(self): + localctx = OmegaConfGrammarParser.PrimitiveContext(self, self._ctx, self.state) + self.enterRule(localctx, 28, self.RULE_primitive) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 171 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 171 + self._errHandler.sync(self) + token = self._input.LA(1) + if token in [OmegaConfGrammarParser.ID]: + self.state = 161 + self.match(OmegaConfGrammarParser.ID) + pass + elif token in [OmegaConfGrammarParser.NULL]: + self.state = 162 + self.match(OmegaConfGrammarParser.NULL) + pass + elif token in [OmegaConfGrammarParser.INT]: + self.state = 163 + self.match(OmegaConfGrammarParser.INT) + pass + elif token in [OmegaConfGrammarParser.FLOAT]: + self.state = 164 + self.match(OmegaConfGrammarParser.FLOAT) + pass + elif token in [OmegaConfGrammarParser.BOOL]: + self.state = 165 + self.match(OmegaConfGrammarParser.BOOL) + pass + elif token in [OmegaConfGrammarParser.UNQUOTED_CHAR]: + self.state = 166 + self.match(OmegaConfGrammarParser.UNQUOTED_CHAR) + pass + elif token in [OmegaConfGrammarParser.COLON]: + self.state = 167 + self.match(OmegaConfGrammarParser.COLON) + pass + elif token in [OmegaConfGrammarParser.ESC]: + self.state = 168 + self.match(OmegaConfGrammarParser.ESC) + pass + elif token in [OmegaConfGrammarParser.WS]: + self.state = 169 + self.match(OmegaConfGrammarParser.WS) + pass + elif token in [OmegaConfGrammarParser.INTER_OPEN]: + self.state = 170 + self.interpolation() + pass + else: + raise NoViableAltException(self) + + self.state = 173 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not ( + ( + ((_la) & ~0x3F) == 0 + and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.INTER_OPEN) + | (1 << OmegaConfGrammarParser.COLON) + | (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) + != 0 + ) + ): + break + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx + + class DictKeyContext(ParserRuleContext): + __slots__ = "parser" + + def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): + super().__init__(parent, invokingState) + self.parser = parser + + def ID(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ID) + else: + return self.getToken(OmegaConfGrammarParser.ID, i) + + def NULL(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.NULL) + else: + return self.getToken(OmegaConfGrammarParser.NULL, i) + + def INT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.INT) + else: + return self.getToken(OmegaConfGrammarParser.INT, i) + + def FLOAT(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.FLOAT) + else: + return self.getToken(OmegaConfGrammarParser.FLOAT, i) + + def BOOL(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.BOOL) + else: + return self.getToken(OmegaConfGrammarParser.BOOL, i) + + def UNQUOTED_CHAR(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.UNQUOTED_CHAR) + else: + return self.getToken(OmegaConfGrammarParser.UNQUOTED_CHAR, i) + + def ESC(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.ESC) + else: + return self.getToken(OmegaConfGrammarParser.ESC, i) + + def WS(self, i: int = None): + if i is None: + return self.getTokens(OmegaConfGrammarParser.WS) + else: + return self.getToken(OmegaConfGrammarParser.WS, i) + + def getRuleIndex(self): + return OmegaConfGrammarParser.RULE_dictKey + + def enterRule(self, listener: ParseTreeListener): + if hasattr(listener, "enterDictKey"): + listener.enterDictKey(self) + + def exitRule(self, listener: ParseTreeListener): + if hasattr(listener, "exitDictKey"): + listener.exitDictKey(self) + + def accept(self, visitor: ParseTreeVisitor): + if hasattr(visitor, "visitDictKey"): + return visitor.visitDictKey(self) + else: + return visitor.visitChildren(self) + + def dictKey(self): + localctx = OmegaConfGrammarParser.DictKeyContext(self, self._ctx, self.state) + self.enterRule(localctx, 30, self.RULE_dictKey) + self._la = 0 # Token type + try: + self.enterOuterAlt(localctx, 1) + self.state = 176 + self._errHandler.sync(self) + _la = self._input.LA(1) + while True: + self.state = 175 + _la = self._input.LA(1) + if not ( + ( + ((_la) & ~0x3F) == 0 + and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) + != 0 + ) + ): + self._errHandler.recoverInline(self) + else: + self._errHandler.reportMatch(self) + self.consume() + self.state = 178 + self._errHandler.sync(self) + _la = self._input.LA(1) + if not ( + ( + ((_la) & ~0x3F) == 0 + and ( + (1 << _la) + & ( + (1 << OmegaConfGrammarParser.FLOAT) + | (1 << OmegaConfGrammarParser.INT) + | (1 << OmegaConfGrammarParser.BOOL) + | (1 << OmegaConfGrammarParser.NULL) + | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) + | (1 << OmegaConfGrammarParser.ID) + | (1 << OmegaConfGrammarParser.ESC) + | (1 << OmegaConfGrammarParser.WS) + ) + ) + != 0 + ) + ): + break + + except RecognitionException as re: + localctx.exception = re + self._errHandler.reportError(self, re) + self._errHandler.recover(self, re) + finally: + self.exitRule() + return localctx diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens b/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens new file mode 100644 index 000000000..c22963b90 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens @@ -0,0 +1,31 @@ +ANY_STR=1 +ESC_INTER=2 +TOP_ESC=3 +INTER_OPEN=4 +BRACE_OPEN=5 +BRACE_CLOSE=6 +QUOTE_OPEN_SINGLE=7 +QUOTE_OPEN_DOUBLE=8 +COMMA=9 +BRACKET_OPEN=10 +BRACKET_CLOSE=11 +COLON=12 +FLOAT=13 +INT=14 +BOOL=15 +NULL=16 +UNQUOTED_CHAR=17 +ID=18 +ESC=19 +WS=20 +INTER_CLOSE=21 +DOT=22 +INTER_KEY=23 +MATCHING_QUOTE_CLOSE=24 +QUOTED_ESC=25 +DOLLAR=26 +INTER_BRACKET_OPEN=27 +INTER_BRACKET_CLOSE=28 +'.'=22 +'['=27 +']'=28 diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py b/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py new file mode 100644 index 000000000..282d60ac6 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py @@ -0,0 +1,141 @@ +# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 +from antlr4 import * + +if __name__ is not None and "." in __name__: + from .OmegaConfGrammarParser import OmegaConfGrammarParser +else: + from OmegaConfGrammarParser import OmegaConfGrammarParser + + +# This class defines a complete listener for a parse tree produced by OmegaConfGrammarParser. +class OmegaConfGrammarParserListener(ParseTreeListener): + # Enter a parse tree produced by OmegaConfGrammarParser#configValue. + def enterConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#configValue. + def exitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#singleElement. + def enterSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#singleElement. + def exitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#text. + def enterText(self, ctx: OmegaConfGrammarParser.TextContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#text. + def exitText(self, ctx: OmegaConfGrammarParser.TextContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#element. + def enterElement(self, ctx: OmegaConfGrammarParser.ElementContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#element. + def exitElement(self, ctx: OmegaConfGrammarParser.ElementContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#listContainer. + def enterListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#listContainer. + def exitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#dictContainer. + def enterDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#dictContainer. + def exitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. + def enterDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. + def exitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#sequence. + def enterSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#sequence. + def exitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#interpolation. + def enterInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#interpolation. + def exitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#interpolationNode. + def enterInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#interpolationNode. + def exitInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#interpolationResolver. + def enterInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#interpolationResolver. + def exitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#configKey. + def enterConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#configKey. + def exitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#resolverName. + def enterResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#resolverName. + def exitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#quotedValue. + def enterQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#quotedValue. + def exitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#primitive. + def enterPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#primitive. + def exitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): + pass + + # Enter a parse tree produced by OmegaConfGrammarParser#dictKey. + def enterDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): + pass + + # Exit a parse tree produced by OmegaConfGrammarParser#dictKey. + def exitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): + pass + + +del OmegaConfGrammarParser diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py b/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py new file mode 100644 index 000000000..30e15f0b9 --- /dev/null +++ b/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py @@ -0,0 +1,78 @@ +# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 +from antlr4 import * + +if __name__ is not None and "." in __name__: + from .OmegaConfGrammarParser import OmegaConfGrammarParser +else: + from OmegaConfGrammarParser import OmegaConfGrammarParser + +# This class defines a complete generic visitor for a parse tree produced by OmegaConfGrammarParser. + + +class OmegaConfGrammarParserVisitor(ParseTreeVisitor): + # Visit a parse tree produced by OmegaConfGrammarParser#configValue. + def visitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#singleElement. + def visitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#text. + def visitText(self, ctx: OmegaConfGrammarParser.TextContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#element. + def visitElement(self, ctx: OmegaConfGrammarParser.ElementContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#listContainer. + def visitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#dictContainer. + def visitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. + def visitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#sequence. + def visitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#interpolation. + def visitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#interpolationNode. + def visitInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#interpolationResolver. + def visitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#configKey. + def visitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#resolverName. + def visitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#quotedValue. + def visitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#primitive. + def visitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): + return self.visitChildren(ctx) + + # Visit a parse tree produced by OmegaConfGrammarParser#dictKey. + def visitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): + return self.visitChildren(ctx) + + +del OmegaConfGrammarParser From a7944a6975b287ecbae353adb08b0d89e85eee6d Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:12:33 +0200 Subject: [PATCH 15/37] change route of module oc --- kapitan/{omegaconf.py => omegaconf_inv.py} | 2 +- kapitan/resources.py | 4 +- omegaconf/__init__.py | 65 - omegaconf/_impl.py | 101 - omegaconf/_utils.py | 1011 ---------- omegaconf/base.py | 947 --------- omegaconf/basecontainer.py | 916 --------- omegaconf/dictconfig.py | 727 ------- omegaconf/errors.py | 141 -- omegaconf/grammar/OmegaConfGrammarLexer.g4 | 137 -- omegaconf/grammar/OmegaConfGrammarParser.g4 | 91 - omegaconf/grammar/__init__.py | 0 .../grammar/gen/OmegaConfGrammarLexer.interp | 130 -- .../grammar/gen/OmegaConfGrammarLexer.py | 404 ---- .../grammar/gen/OmegaConfGrammarLexer.tokens | 31 - .../grammar/gen/OmegaConfGrammarParser.interp | 83 - .../grammar/gen/OmegaConfGrammarParser.py | 1745 ----------------- .../grammar/gen/OmegaConfGrammarParser.tokens | 31 - .../gen/OmegaConfGrammarParserListener.py | 141 -- .../gen/OmegaConfGrammarParserVisitor.py | 78 - omegaconf/grammar/gen/__init__.py | 0 omegaconf/grammar_parser.py | 140 -- omegaconf/grammar_visitor.py | 362 ---- omegaconf/listconfig.py | 643 ------ omegaconf/nodes.py | 513 ----- omegaconf/omegaconf.py | 1144 ----------- omegaconf/py.typed | 0 omegaconf/resolvers/__init__.py | 5 - omegaconf/resolvers/oc/__init__.py | 109 - omegaconf/resolvers/oc/dict.py | 78 - omegaconf/version.py | 13 - 31 files changed, 3 insertions(+), 9789 deletions(-) rename kapitan/{omegaconf.py => omegaconf_inv.py} (98%) delete mode 100644 omegaconf/__init__.py delete mode 100644 omegaconf/_impl.py delete mode 100644 omegaconf/_utils.py delete mode 100644 omegaconf/base.py delete mode 100644 omegaconf/basecontainer.py delete mode 100644 omegaconf/dictconfig.py delete mode 100644 omegaconf/errors.py delete mode 100644 omegaconf/grammar/OmegaConfGrammarLexer.g4 delete mode 100644 omegaconf/grammar/OmegaConfGrammarParser.g4 delete mode 100644 omegaconf/grammar/__init__.py delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.interp delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.py delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.interp delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.py delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParser.tokens delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParserListener.py delete mode 100644 omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py delete mode 100644 omegaconf/grammar/gen/__init__.py delete mode 100644 omegaconf/grammar_parser.py delete mode 100644 omegaconf/grammar_visitor.py delete mode 100644 omegaconf/listconfig.py delete mode 100644 omegaconf/nodes.py delete mode 100644 omegaconf/omegaconf.py delete mode 100644 omegaconf/py.typed delete mode 100644 omegaconf/resolvers/__init__.py delete mode 100644 omegaconf/resolvers/oc/__init__.py delete mode 100644 omegaconf/resolvers/oc/dict.py delete mode 100644 omegaconf/version.py diff --git a/kapitan/omegaconf.py b/kapitan/omegaconf_inv.py similarity index 98% rename from kapitan/omegaconf.py rename to kapitan/omegaconf_inv.py index ebafd09ef..67cc9ba5b 100644 --- a/kapitan/omegaconf.py +++ b/kapitan/omegaconf_inv.py @@ -8,7 +8,7 @@ import time from kapitan.errors import InventoryError -from omegaconf import Node, OmegaConf, errors +from kapitan.oc.omegaconf import Node, OmegaConf, errors logger = logging.getLogger(__name__) diff --git a/kapitan/resources.py b/kapitan/resources.py index 8fe05094a..22f9b71c9 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -26,9 +26,9 @@ from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError from kapitan.migrate_omegaconf import migrate -from kapitan.omegaconf import inventory_omegaconf +from kapitan.omegaconf_inv import inventory_omegaconf from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -from omegaconf import errors +from kapitan.oc.omegaconf import errors logger = logging.getLogger(__name__) diff --git a/omegaconf/__init__.py b/omegaconf/__init__.py deleted file mode 100644 index 1670cf08f..000000000 --- a/omegaconf/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -from .base import Container, DictKeyType, Node, SCMode, UnionNode -from .dictconfig import DictConfig -from .errors import ( - KeyValidationError, - MissingMandatoryValue, - ReadonlyConfigError, - UnsupportedValueType, - ValidationError, -) -from .listconfig import ListConfig -from .nodes import ( - AnyNode, - BooleanNode, - BytesNode, - EnumNode, - FloatNode, - IntegerNode, - PathNode, - StringNode, - ValueNode, -) -from .omegaconf import ( - II, - MISSING, - SI, - OmegaConf, - Resolver, - flag_override, - open_dict, - read_write, -) -from .version import __version__ - -__all__ = [ - "__version__", - "MissingMandatoryValue", - "ValidationError", - "ReadonlyConfigError", - "UnsupportedValueType", - "KeyValidationError", - "Container", - "UnionNode", - "ListConfig", - "DictConfig", - "DictKeyType", - "OmegaConf", - "Resolver", - "SCMode", - "flag_override", - "read_write", - "open_dict", - "Node", - "ValueNode", - "AnyNode", - "IntegerNode", - "StringNode", - "BytesNode", - "PathNode", - "BooleanNode", - "EnumNode", - "FloatNode", - "MISSING", - "SI", - "II", -] diff --git a/omegaconf/_impl.py b/omegaconf/_impl.py deleted file mode 100644 index 0c30ef6a2..000000000 --- a/omegaconf/_impl.py +++ /dev/null @@ -1,101 +0,0 @@ -from typing import Any - -from omegaconf import MISSING, Container, DictConfig, ListConfig, Node, ValueNode -from omegaconf.errors import ConfigTypeError, InterpolationToMissingValueError - -from ._utils import _DEFAULT_MARKER_, _get_value - - -def _resolve_container_value(cfg: Container, key: Any) -> None: - node = cfg._get_child(key) - assert isinstance(node, Node) - if node._is_interpolation(): - try: - resolved = node._dereference_node() - except InterpolationToMissingValueError: - node._set_value(MISSING) - else: - if isinstance(resolved, Container): - _resolve(resolved) - if isinstance(resolved, Container) and isinstance(node, ValueNode): - cfg[key] = resolved - else: - node._set_value(_get_value(resolved)) - else: - _resolve(node) - - -def _resolve(cfg: Node) -> Node: - assert isinstance(cfg, Node) - if cfg._is_interpolation(): - try: - resolved = cfg._dereference_node() - except InterpolationToMissingValueError: - cfg._set_value(MISSING) - else: - cfg._set_value(resolved._value()) - - if isinstance(cfg, DictConfig): - for k in cfg.keys(): - _resolve_container_value(cfg, k) - - elif isinstance(cfg, ListConfig): - for i in range(len(cfg)): - _resolve_container_value(cfg, i) - - return cfg - - -def select_value( - cfg: Container, - key: str, - *, - default: Any = _DEFAULT_MARKER_, - throw_on_resolution_failure: bool = True, - throw_on_missing: bool = False, - absolute_key: bool = False, -) -> Any: - node = select_node( - cfg=cfg, - key=key, - throw_on_resolution_failure=throw_on_resolution_failure, - throw_on_missing=throw_on_missing, - absolute_key=absolute_key, - ) - - node_not_found = node is None - if node_not_found or node._is_missing(): - if default is not _DEFAULT_MARKER_: - return default - else: - return None - - return _get_value(node) - - -def select_node( - cfg: Container, - key: str, - *, - throw_on_resolution_failure: bool = True, - throw_on_missing: bool = False, - absolute_key: bool = False, -) -> Any: - try: - # for non relative keys, the interpretation can be: - # 1. relative to cfg - # 2. relative to the config root - # This is controlled by the absolute_key flag. By default, such keys are relative to cfg. - if not absolute_key and not key.startswith("."): - key = f".{key}" - - cfg, key = cfg._resolve_key_and_root(key) - _root, _last_key, node = cfg._select_impl( - key, - throw_on_missing=throw_on_missing, - throw_on_resolution_failure=throw_on_resolution_failure, - ) - except ConfigTypeError: - return None - - return node diff --git a/omegaconf/_utils.py b/omegaconf/_utils.py deleted file mode 100644 index 3339c1a21..000000000 --- a/omegaconf/_utils.py +++ /dev/null @@ -1,1011 +0,0 @@ -import copy -import os -import pathlib -import re -import string -import sys -import types -import warnings -from contextlib import contextmanager -from enum import Enum -from textwrap import dedent -from typing import ( - Any, - Dict, - Iterator, - List, - Optional, - Tuple, - Type, - Union, - get_type_hints, -) - -import yaml - -from .errors import ( - ConfigIndexError, - ConfigTypeError, - ConfigValueError, - GrammarParseError, - OmegaConfBaseException, - ValidationError, -) -from .grammar_parser import SIMPLE_INTERPOLATION_PATTERN, parse - -try: - import dataclasses - -except ImportError: # pragma: no cover - dataclasses = None # type: ignore # pragma: no cover - -try: - import attr - -except ImportError: # pragma: no cover - attr = None # type: ignore # pragma: no cover - -NoneType: Type[None] = type(None) - -BUILTIN_VALUE_TYPES: Tuple[Type[Any], ...] = ( - int, - float, - bool, - str, - bytes, - NoneType, -) - -# Regexprs to match key paths like: a.b, a[b], ..a[c].d, etc. -# We begin by matching the head (in these examples: a, a, ..a). -# This can be read as "dots followed by any character but `.` or `[`" -# Note that a key starting with brackets, like [a], is purposedly *not* -# matched here and will instead be handled in the next regex below (this -# is to keep this regex simple). -KEY_PATH_HEAD = re.compile(r"(\.)*[^.[]*") -# Then we match other keys. The following expression matches one key and can -# be read as a choice between two syntaxes: -# - `.` followed by anything except `.` or `[` (ex: .b, .d) -# - `[` followed by anything then `]` (ex: [b], [c]) -KEY_PATH_OTHER = re.compile(r"\.([^.[]*)|\[(.*?)\]") - - -# source: https://yaml.org/type/bool.html -YAML_BOOL_TYPES = [ - "y", - "Y", - "yes", - "Yes", - "YES", - "n", - "N", - "no", - "No", - "NO", - "true", - "True", - "TRUE", - "false", - "False", - "FALSE", - "on", - "On", - "ON", - "off", - "Off", - "OFF", -] - - -class Marker: - def __init__(self, desc: str): - self.desc = desc - - def __repr__(self) -> str: - return self.desc - - -# To be used as default value when `None` is not an option. -_DEFAULT_MARKER_: Any = Marker("_DEFAULT_MARKER_") - - -class OmegaConfDumper(yaml.Dumper): # type: ignore - str_representer_added = False - - @staticmethod - def str_representer(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode: - with_quotes = yaml_is_bool(data) or is_int(data) or is_float(data) - return dumper.represent_scalar( - yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG, - data, - style=("'" if with_quotes else None), - ) - - -def get_omega_conf_dumper() -> Type[OmegaConfDumper]: - if not OmegaConfDumper.str_representer_added: - OmegaConfDumper.add_representer(str, OmegaConfDumper.str_representer) - OmegaConfDumper.str_representer_added = True - return OmegaConfDumper - - -def yaml_is_bool(b: str) -> bool: - return b in YAML_BOOL_TYPES - - -def get_yaml_loader() -> Any: - class OmegaConfLoader(yaml.SafeLoader): # type: ignore - def construct_mapping(self, node: yaml.Node, deep: bool = False) -> Any: - keys = set() - for key_node, value_node in node.value: - if key_node.tag != yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG: - continue - if key_node.value in keys: - raise yaml.constructor.ConstructorError( - "while constructing a mapping", - node.start_mark, - f"found duplicate key {key_node.value}", - key_node.start_mark, - ) - keys.add(key_node.value) - return super().construct_mapping(node, deep=deep) - - loader = OmegaConfLoader - loader.add_implicit_resolver( - "tag:yaml.org,2002:float", - re.compile( - """^(?: - [-+]?[0-9]+(?:_[0-9]+)*\\.[0-9_]*(?:[eE][-+]?[0-9]+)? - |[-+]?[0-9]+(?:_[0-9]+)*(?:[eE][-+]?[0-9]+) - |\\.[0-9]+(?:_[0-9]+)*(?:[eE][-+][0-9]+)? - |[-+]?[0-9]+(?:_[0-9]+)*(?::[0-5]?[0-9])+\\.[0-9_]* - |[-+]?\\.(?:inf|Inf|INF) - |\\.(?:nan|NaN|NAN))$""", - re.X, - ), - list("-+0123456789."), - ) - loader.yaml_implicit_resolvers = { - key: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:timestamp"] - for key, resolvers in loader.yaml_implicit_resolvers.items() - } - - loader.add_constructor( - "tag:yaml.org,2002:python/object/apply:pathlib.Path", - lambda loader, node: pathlib.Path(*loader.construct_sequence(node)), - ) - loader.add_constructor( - "tag:yaml.org,2002:python/object/apply:pathlib.PosixPath", - lambda loader, node: pathlib.PosixPath(*loader.construct_sequence(node)), - ) - loader.add_constructor( - "tag:yaml.org,2002:python/object/apply:pathlib.WindowsPath", - lambda loader, node: pathlib.WindowsPath(*loader.construct_sequence(node)), - ) - - return loader - - -def _get_class(path: str) -> type: - from importlib import import_module - - module_path, _, class_name = path.rpartition(".") - mod = import_module(module_path) - try: - klass: type = getattr(mod, class_name) - except AttributeError: - raise ImportError(f"Class {class_name} is not in module {module_path}") - return klass - - -def is_union_annotation(type_: Any) -> bool: - if sys.version_info >= (3, 10): # pragma: no cover - if isinstance(type_, types.UnionType): - return True - return getattr(type_, "__origin__", None) is Union - - -def _resolve_optional(type_: Any) -> Tuple[bool, Any]: - """Check whether `type_` is equivalent to `typing.Optional[T]` for some T.""" - if is_union_annotation(type_): - args = type_.__args__ - if NoneType in args: - optional = True - args = tuple(a for a in args if a is not NoneType) - else: - optional = False - if len(args) == 1: - return optional, args[0] - elif len(args) >= 2: - return optional, Union[args] - else: - assert False - - if type_ is Any: - return True, Any - - if type_ in (None, NoneType): - return True, NoneType - - return False, type_ - - -def _is_optional(obj: Any, key: Optional[Union[int, str]] = None) -> bool: - """Check `obj` metadata to see if the given node is optional.""" - from .base import Container, Node - - if key is not None: - assert isinstance(obj, Container) - obj = obj._get_node(key) - assert isinstance(obj, Node) - return obj._is_optional() - - -def _resolve_forward(type_: Type[Any], module: str) -> Type[Any]: - import typing # lgtm [py/import-and-import-from] - - forward = typing.ForwardRef if hasattr(typing, "ForwardRef") else typing._ForwardRef # type: ignore - if type(type_) is forward: - return _get_class(f"{module}.{type_.__forward_arg__}") - else: - if is_dict_annotation(type_): - kt, vt = get_dict_key_value_types(type_) - if kt is not None: - kt = _resolve_forward(kt, module=module) - if vt is not None: - vt = _resolve_forward(vt, module=module) - return Dict[kt, vt] # type: ignore - if is_list_annotation(type_): - et = get_list_element_type(type_) - if et is not None: - et = _resolve_forward(et, module=module) - return List[et] # type: ignore - if is_tuple_annotation(type_): - its = get_tuple_item_types(type_) - its = tuple(_resolve_forward(it, module=module) for it in its) - return Tuple[its] # type: ignore - - return type_ - - -def extract_dict_subclass_data(obj: Any, parent: Any) -> Optional[Dict[str, Any]]: - """Check if obj is an instance of a subclass of Dict. If so, extract the Dict keys/values.""" - from omegaconf.omegaconf import _maybe_wrap - - is_type = isinstance(obj, type) - obj_type = obj if is_type else type(obj) - subclasses_dict = is_dict_subclass(obj_type) - - if subclasses_dict: - warnings.warn( - f"Class `{obj_type.__name__}` subclasses `Dict`." - + " Subclassing `Dict` in Structured Config classes is deprecated," - + " see github.com/omry/omegaconf/issues/663", - UserWarning, - stacklevel=9, - ) - - if is_type: - return None - elif subclasses_dict: - dict_subclass_data = {} - key_type, element_type = get_dict_key_value_types(obj_type) - for name, value in obj.items(): - is_optional, type_ = _resolve_optional(element_type) - type_ = _resolve_forward(type_, obj.__module__) - try: - dict_subclass_data[name] = _maybe_wrap( - ref_type=type_, - is_optional=is_optional, - key=name, - value=value, - parent=parent, - ) - except ValidationError as ex: - format_and_raise(node=None, key=name, value=value, cause=ex, msg=str(ex)) - return dict_subclass_data - else: - return None - - -def get_attr_class_fields(obj: Any) -> List["attr.Attribute[Any]"]: - is_type = isinstance(obj, type) - obj_type = obj if is_type else type(obj) - fields = attr.fields_dict(obj_type).values() - return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True] - - -def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: - from omegaconf.omegaconf import OmegaConf, _maybe_wrap - - flags = {"allow_objects": allow_objects} if allow_objects is not None else {} - - from omegaconf import MISSING - - d = {} - is_type = isinstance(obj, type) - obj_type = obj if is_type else type(obj) - dummy_parent = OmegaConf.create({}, flags=flags) - dummy_parent._metadata.object_type = obj_type - resolved_hints = get_type_hints(obj_type) - - for attrib in get_attr_class_fields(obj): - name = attrib.name - is_optional, type_ = _resolve_optional(resolved_hints[name]) - type_ = _resolve_forward(type_, obj.__module__) - if not is_type: - value = getattr(obj, name) - else: - value = attrib.default - if value == attr.NOTHING: - value = MISSING - if is_union_annotation(type_) and not is_supported_union_annotation(type_): - e = ConfigValueError(f"Unions of containers are not supported:\n{name}: {type_str(type_)}") - format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) - - try: - d[name] = _maybe_wrap( - ref_type=type_, - is_optional=is_optional, - key=name, - value=value, - parent=dummy_parent, - ) - except (ValidationError, GrammarParseError) as ex: - format_and_raise(node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)) - d[name]._set_parent(None) - dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent) - if dict_subclass_data is not None: - d.update(dict_subclass_data) - return d - - -def get_dataclass_fields(obj: Any) -> List["dataclasses.Field[Any]"]: - fields = dataclasses.fields(obj) - return [f for f in fields if f.metadata.get("omegaconf_ignore") is not True] - - -def get_dataclass_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: - from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap - - flags = {"allow_objects": allow_objects} if allow_objects is not None else {} - d = {} - is_type = isinstance(obj, type) - obj_type = get_type_of(obj) - dummy_parent = OmegaConf.create({}, flags=flags) - dummy_parent._metadata.object_type = obj_type - resolved_hints = get_type_hints(obj_type) - for field in get_dataclass_fields(obj): - name = field.name - is_optional, type_ = _resolve_optional(resolved_hints[field.name]) - type_ = _resolve_forward(type_, obj.__module__) - has_default = field.default != dataclasses.MISSING - has_default_factory = field.default_factory != dataclasses.MISSING - - if not is_type: - value = getattr(obj, name) - else: - if has_default: - value = field.default - elif has_default_factory: - value = field.default_factory() # type: ignore - else: - value = MISSING - - if is_union_annotation(type_) and not is_supported_union_annotation(type_): - e = ConfigValueError(f"Unions of containers are not supported:\n{name}: {type_str(type_)}") - format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) - try: - d[name] = _maybe_wrap( - ref_type=type_, - is_optional=is_optional, - key=name, - value=value, - parent=dummy_parent, - ) - except (ValidationError, GrammarParseError) as ex: - format_and_raise(node=dummy_parent, key=name, value=value, cause=ex, msg=str(ex)) - d[name]._set_parent(None) - dict_subclass_data = extract_dict_subclass_data(obj=obj, parent=dummy_parent) - if dict_subclass_data is not None: - d.update(dict_subclass_data) - return d - - -def is_dataclass(obj: Any) -> bool: - from omegaconf.base import Node - - if dataclasses is None or isinstance(obj, Node): - return False - is_dataclass = dataclasses.is_dataclass(obj) - assert isinstance(is_dataclass, bool) - return is_dataclass - - -def is_attr_class(obj: Any) -> bool: - from omegaconf.base import Node - - if attr is None or isinstance(obj, Node): - return False - return attr.has(obj) - - -def is_structured_config(obj: Any) -> bool: - return is_attr_class(obj) or is_dataclass(obj) - - -def is_dataclass_frozen(type_: Any) -> bool: - return type_.__dataclass_params__.frozen # type: ignore - - -def is_attr_frozen(type_: type) -> bool: - # This is very hacky and probably fragile as well. - # Unfortunately currently there isn't an official API in attr that can detect that. - # noinspection PyProtectedMember - return type_.__setattr__ == attr._make._frozen_setattrs # type: ignore - - -def get_type_of(class_or_object: Any) -> Type[Any]: - type_ = class_or_object - if not isinstance(type_, type): - type_ = type(class_or_object) - assert isinstance(type_, type) - return type_ - - -def is_structured_config_frozen(obj: Any) -> bool: - type_ = get_type_of(obj) - - if is_dataclass(type_): - return is_dataclass_frozen(type_) - if is_attr_class(type_): - return is_attr_frozen(type_) - return False - - -def get_structured_config_init_field_names(obj: Any) -> List[str]: - fields: Union[List["dataclasses.Field[Any]"], List["attr.Attribute[Any]"]] - if is_dataclass(obj): - fields = get_dataclass_fields(obj) - elif is_attr_class(obj): - fields = get_attr_class_fields(obj) - else: - raise ValueError(f"Unsupported type: {type(obj).__name__}") - return [f.name for f in fields if f.init] - - -def get_structured_config_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: - if is_dataclass(obj): - return get_dataclass_data(obj, allow_objects=allow_objects) - elif is_attr_class(obj): - return get_attr_data(obj, allow_objects=allow_objects) - else: - raise ValueError(f"Unsupported type: {type(obj).__name__}") - - -class ValueKind(Enum): - VALUE = 0 - MANDATORY_MISSING = 1 - INTERPOLATION = 2 - - -def _is_missing_value(value: Any) -> bool: - from omegaconf import Node - - if isinstance(value, Node): - value = value._value() - return _is_missing_literal(value) - - -def _is_missing_literal(value: Any) -> bool: - # Uses literal '???' instead of the MISSING const for performance reasons. - return isinstance(value, str) and value == "???" - - -def _is_none(value: Any, resolve: bool = False, throw_on_resolution_failure: bool = True) -> bool: - from omegaconf import Node - - if not isinstance(value, Node): - return value is None - - if resolve: - value = value._maybe_dereference_node(throw_on_resolution_failure=throw_on_resolution_failure) - if not throw_on_resolution_failure and value is None: - # Resolution failure: consider that it is *not* None. - return False - assert isinstance(value, Node) - - return value._is_none() - - -def get_value_kind(value: Any, strict_interpolation_validation: bool = False) -> ValueKind: - """ - Determine the kind of a value - Examples: - VALUE: "10", "20", True - MANDATORY_MISSING: "???" - INTERPOLATION: "${foo.bar}", "${foo.${bar}}", "${foo:bar}", "[${foo}, ${bar}]", - "ftp://${host}/path", "${foo:${bar}, [true], {'baz': ${baz}}}" - - :param value: Input to classify. - :param strict_interpolation_validation: If `True`, then when `value` is a string - containing "${", it is parsed to validate the interpolation syntax. If `False`, - this parsing step is skipped: this is more efficient, but will not detect errors. - """ - - if _is_missing_value(value): - return ValueKind.MANDATORY_MISSING - - if _is_interpolation(value, strict_interpolation_validation): - return ValueKind.INTERPOLATION - - return ValueKind.VALUE - - -def _is_interpolation(v: Any, strict_interpolation_validation: bool = False) -> bool: - from omegaconf import Node - - if isinstance(v, Node): - v = v._value() - - if isinstance(v, str) and _is_interpolation_string(v, strict_interpolation_validation): - return True - return False - - -def _is_interpolation_string(value: str, strict_interpolation_validation: bool) -> bool: - # We identify potential interpolations by the presence of "${" in the string. - # Note that escaped interpolations (ex: "esc: \${bar}") are identified as - # interpolations: this is intended, since they must be processed as interpolations - # for the string to be properly un-escaped. - # Keep in mind that invalid interpolations will only be detected when - # `strict_interpolation_validation` is True. - if "${" in value: - if strict_interpolation_validation: - # First try the cheap regex matching that detects common interpolations. - if SIMPLE_INTERPOLATION_PATTERN.match(value) is None: - # If no match, do the more expensive grammar parsing to detect errors. - parse(value) - return True - return False - - -def _is_special(value: Any) -> bool: - """Special values are None, MISSING, and interpolation.""" - return _is_none(value) or get_value_kind(value) in ( - ValueKind.MANDATORY_MISSING, - ValueKind.INTERPOLATION, - ) - - -def is_float(st: str) -> bool: - try: - float(st) - return True - except ValueError: - return False - - -def is_int(st: str) -> bool: - try: - int(st) - return True - except ValueError: - return False - - -def is_primitive_list(obj: Any) -> bool: - return isinstance(obj, (list, tuple)) - - -def is_primitive_dict(obj: Any) -> bool: - t = get_type_of(obj) - return t is dict - - -def is_dict_annotation(type_: Any) -> bool: - if type_ in (dict, Dict): - return True - origin = getattr(type_, "__origin__", None) - # type_dict is a bit hard to detect. - # this support is tentative, if it eventually causes issues in other areas it may be dropped. - if sys.version_info < (3, 7, 0): # pragma: no cover - typed_dict = hasattr(type_, "__base__") and type_.__base__ == Dict - return origin is Dict or type_ is Dict or typed_dict - else: # pragma: no cover - typed_dict = hasattr(type_, "__base__") and type_.__base__ == dict - return origin is dict or typed_dict - - -def is_list_annotation(type_: Any) -> bool: - if type_ in (list, List): - return True - origin = getattr(type_, "__origin__", None) - if sys.version_info < (3, 7, 0): - return origin is List or type_ is List # pragma: no cover - else: - return origin is list # pragma: no cover - - -def is_tuple_annotation(type_: Any) -> bool: - if type_ in (tuple, Tuple): - return True - origin = getattr(type_, "__origin__", None) - if sys.version_info < (3, 7, 0): - return origin is Tuple or type_ is Tuple # pragma: no cover - else: - return origin is tuple # pragma: no cover - - -def is_supported_union_annotation(obj: Any) -> bool: - """Currently only primitive types are supported in Unions, e.g. Union[int, str]""" - if not is_union_annotation(obj): - return False - args = obj.__args__ - return all(is_primitive_type_annotation(arg) for arg in args) - - -def is_dict_subclass(type_: Any) -> bool: - return type_ is not None and isinstance(type_, type) and issubclass(type_, Dict) - - -def is_dict(obj: Any) -> bool: - return is_primitive_dict(obj) or is_dict_annotation(obj) or is_dict_subclass(obj) - - -def is_primitive_container(obj: Any) -> bool: - return is_primitive_list(obj) or is_primitive_dict(obj) - - -def get_list_element_type(ref_type: Optional[Type[Any]]) -> Any: - args = getattr(ref_type, "__args__", None) - if ref_type is not List and args is not None and args[0]: - element_type = args[0] - else: - element_type = Any - return element_type - - -def get_tuple_item_types(ref_type: Type[Any]) -> Tuple[Any, ...]: - args = getattr(ref_type, "__args__", None) - if args in (None, ()): - args = (Any, ...) - assert isinstance(args, tuple) - return args - - -def get_dict_key_value_types(ref_type: Any) -> Tuple[Any, Any]: - args = getattr(ref_type, "__args__", None) - if args is None: - bases = getattr(ref_type, "__orig_bases__", None) - if bases is not None and len(bases) > 0: - args = getattr(bases[0], "__args__", None) - - key_type: Any - element_type: Any - if ref_type is None or ref_type == Dict: - key_type = Any - element_type = Any - else: - if args is not None: - key_type = args[0] - element_type = args[1] - else: - key_type = Any - element_type = Any - - return key_type, element_type - - -def is_valid_value_annotation(type_: Any) -> bool: - _, type_ = _resolve_optional(type_) - return ( - type_ is Any - or is_primitive_type_annotation(type_) - or is_structured_config(type_) - or is_container_annotation(type_) - or is_supported_union_annotation(type_) - ) - - -def _valid_dict_key_annotation_type(type_: Any) -> bool: - from omegaconf import DictKeyType - - return type_ is None or type_ is Any or issubclass(type_, DictKeyType.__args__) # type: ignore - - -def is_primitive_type_annotation(type_: Any) -> bool: - type_ = get_type_of(type_) - return issubclass(type_, (Enum, pathlib.Path)) or type_ in BUILTIN_VALUE_TYPES - - -def _get_value(value: Any) -> Any: - from .base import Container, UnionNode - from .nodes import ValueNode - - if isinstance(value, ValueNode): - return value._value() - elif isinstance(value, Container): - boxed = value._value() - if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): - return boxed - elif isinstance(value, UnionNode): - boxed = value._value() - if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): - return boxed - else: - return _get_value(boxed) # pass through value of boxed node - - # return primitives and regular OmegaConf Containers as is - return value - - -def get_type_hint(obj: Any, key: Any = None) -> Optional[Type[Any]]: - from omegaconf import Container, Node - - if isinstance(obj, Container): - if key is not None: - obj = obj._get_node(key) - else: - if key is not None: - raise ValueError("Key must only be provided when obj is a container") - - if isinstance(obj, Node): - ref_type = obj._metadata.ref_type - if obj._is_optional() and ref_type is not Any: - return Optional[ref_type] # type: ignore - else: - return ref_type - else: - return Any # type: ignore - - -def _raise(ex: Exception, cause: Exception) -> None: - # Set the environment variable OC_CAUSE=1 to get a stacktrace that includes the - # causing exception. - env_var = os.environ["OC_CAUSE"] if "OC_CAUSE" in os.environ else None - debugging = sys.gettrace() is not None - full_backtrace = (debugging and not env_var == "0") or (env_var == "1") - if full_backtrace: - ex.__cause__ = cause - else: - ex.__cause__ = None - raise ex.with_traceback(sys.exc_info()[2]) # set env var OC_CAUSE=1 for full trace - - -def format_and_raise( - node: Any, - key: Any, - value: Any, - msg: str, - cause: Exception, - type_override: Any = None, -) -> None: - from omegaconf import OmegaConf - from omegaconf.base import Node - - if isinstance(cause, AssertionError): - raise - - if isinstance(cause, OmegaConfBaseException) and cause._initialized: - ex = cause - if type_override is not None: - ex = type_override(str(cause)) - ex.__dict__ = copy.deepcopy(cause.__dict__) - _raise(ex, cause) - - object_type: Optional[Type[Any]] - object_type_str: Optional[str] = None - ref_type: Optional[Type[Any]] - ref_type_str: Optional[str] - - child_node: Optional[Node] = None - if node is None: - full_key = key if key is not None else "" - object_type = None - ref_type = None - ref_type_str = None - else: - if key is not None and not node._is_none(): - child_node = node._get_node(key, validate_access=False) - - try: - full_key = node._get_full_key(key=key) - except Exception as exc: - # Since we are handling an exception, raising a different one here would - # be misleading. Instead, we display it in the key. - full_key = f"" - - object_type = OmegaConf.get_type(node) - object_type_str = type_str(object_type) - - ref_type = get_type_hint(node) - ref_type_str = type_str(ref_type) - - msg = string.Template(msg).safe_substitute( - REF_TYPE=ref_type_str, - OBJECT_TYPE=object_type_str, - KEY=key, - FULL_KEY=full_key, - VALUE=value, - VALUE_TYPE=type_str(type(value), include_module_name=True), - KEY_TYPE=f"{type(key).__name__}", - ) - - if ref_type not in (None, Any): - template = dedent( - """\ - $MSG - full_key: $FULL_KEY - reference_type=$REF_TYPE - object_type=$OBJECT_TYPE""" - ) - else: - template = dedent( - """\ - $MSG - full_key: $FULL_KEY - object_type=$OBJECT_TYPE""" - ) - s = string.Template(template=template) - - message = s.substitute(REF_TYPE=ref_type_str, OBJECT_TYPE=object_type_str, MSG=msg, FULL_KEY=full_key) - exception_type = type(cause) if type_override is None else type_override - if exception_type == TypeError: - exception_type = ConfigTypeError - elif exception_type == IndexError: - exception_type = ConfigIndexError - - ex = exception_type(f"{message}") - if issubclass(exception_type, OmegaConfBaseException): - ex._initialized = True - ex.msg = message - ex.parent_node = node - ex.child_node = child_node - ex.key = key - ex.full_key = full_key - ex.value = value - ex.object_type = object_type - ex.object_type_str = object_type_str - ex.ref_type = ref_type - ex.ref_type_str = ref_type_str - - _raise(ex, cause) - - -def type_str(t: Any, include_module_name: bool = False) -> str: - is_optional, t = _resolve_optional(t) - if t is NoneType: - return str(t.__name__) - if t is Any: - return "Any" - if t is ...: - return "..." - - if hasattr(t, "__name__"): - name = str(t.__name__) - elif getattr(t, "_name", None) is not None: # pragma: no cover - name = str(t._name) - elif getattr(t, "__origin__", None) is not None: # pragma: no cover - name = type_str(t.__origin__) - else: - name = str(t) - if name.startswith("typing."): # pragma: no cover - name = name[len("typing.") :] - - args = getattr(t, "__args__", None) - if args is not None: - args = ", ".join([type_str(t, include_module_name=include_module_name) for t in t.__args__]) - ret = f"{name}[{args}]" - else: - ret = name - if include_module_name: - if ( - hasattr(t, "__module__") - and t.__module__ != "builtins" - and t.__module__ != "typing" - and not t.__module__.startswith("omegaconf.") - ): - module_prefix = str(t.__module__) + "." - else: - module_prefix = "" - ret = module_prefix + ret - if is_optional: - return f"Optional[{ret}]" - else: - return ret - - -def _ensure_container(target: Any, flags: Optional[Dict[str, bool]] = None) -> Any: - from omegaconf import OmegaConf - - if is_primitive_container(target): - assert isinstance(target, (list, dict)) - target = OmegaConf.create(target, flags=flags) - elif is_structured_config(target): - target = OmegaConf.structured(target, flags=flags) - elif not OmegaConf.is_config(target): - raise ValueError( - "Invalid input. Supports one of " - + "[dict,list,DictConfig,ListConfig,dataclass,dataclass instance,attr class,attr class instance]" - ) - - return target - - -def is_generic_list(type_: Any) -> bool: - """ - Checks if a type is a generic list, for example: - list returns False - typing.List returns False - typing.List[T] returns True - - :param type_: variable type - :return: bool - """ - return is_list_annotation(type_) and get_list_element_type(type_) is not None - - -def is_generic_dict(type_: Any) -> bool: - """ - Checks if a type is a generic dict, for example: - list returns False - typing.List returns False - typing.List[T] returns True - - :param type_: variable type - :return: bool - """ - return is_dict_annotation(type_) and len(get_dict_key_value_types(type_)) > 0 - - -def is_container_annotation(type_: Any) -> bool: - return is_list_annotation(type_) or is_dict_annotation(type_) - - -def split_key(key: str) -> List[str]: - """ - Split a full key path into its individual components. - - This is similar to `key.split(".")` but also works with the getitem syntax: - "a.b" -> ["a", "b"] - "a[b]" -> ["a", "b"] - ".a.b[c].d" -> ["", "a", "b", "c", "d"] - "[a].b" -> ["a", "b"] - """ - # Obtain the first part of the key (in docstring examples: a, a, .a, '') - first = KEY_PATH_HEAD.match(key) - assert first is not None - first_stop = first.span()[1] - - # `tokens` will contain all elements composing the key. - tokens = key[0:first_stop].split(".") - - # Optimization in case `key` has no other component: we are done. - if first_stop == len(key): - return tokens - - if key[first_stop] == "[" and not tokens[-1]: - # This is a special case where the first key starts with brackets, e.g. - # [a] or ..[a]. In that case there is an extra "" in `tokens` that we - # need to get rid of: - # [a] -> tokens = [""] but we would like [] - # ..[a] -> tokens = ["", "", ""] but we would like ["", ""] - tokens.pop() - - # Identify other key elements (in docstring examples: b, b, b/c/d, b) - others = KEY_PATH_OTHER.findall(key[first_stop:]) - - # There are two groups in the `KEY_PATH_OTHER` regex: one for keys starting - # with a dot (.b, .d) and one for keys starting with a bracket ([b], [c]). - # Only one group can be non-empty. - tokens += [dot_key if dot_key else bracket_key for dot_key, bracket_key in others] - - return tokens - - -# Similar to Python 3.7+'s `contextlib.nullcontext` (which should be used instead, -# once support for Python 3.6 is dropped). -@contextmanager -def nullcontext(enter_result: Any = None) -> Iterator[Any]: - yield enter_result diff --git a/omegaconf/base.py b/omegaconf/base.py deleted file mode 100644 index d5eff9a0d..000000000 --- a/omegaconf/base.py +++ /dev/null @@ -1,947 +0,0 @@ -import copy -import sys -from abc import ABC, abstractmethod -from collections import defaultdict -from dataclasses import dataclass, field -from enum import Enum -from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, Union - -from antlr4 import ParserRuleContext - -from ._utils import ( - _DEFAULT_MARKER_, - NoneType, - ValueKind, - _get_value, - _is_interpolation, - _is_missing_value, - _is_special, - format_and_raise, - get_value_kind, - is_union_annotation, - is_valid_value_annotation, - split_key, - type_str, -) -from .errors import ( - ConfigKeyError, - ConfigTypeError, - InterpolationKeyError, - InterpolationResolutionError, - InterpolationToMissingValueError, - InterpolationValidationError, - MissingMandatoryValue, - UnsupportedInterpolationType, - ValidationError, -) -from .grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser -from .grammar_parser import parse -from .grammar_visitor import GrammarVisitor - -DictKeyType = Union[str, bytes, int, Enum, float, bool] - - -@dataclass -class Metadata: - ref_type: Union[Type[Any], Any] - - object_type: Union[Type[Any], Any] - - optional: bool - - key: Any - - # Flags have 3 modes: - # unset : inherit from parent (None if no parent specifies) - # set to true: flag is true - # set to false: flag is false - flags: Optional[Dict[str, bool]] = None - - # If True, when checking the value of a flag, if the flag is not set None is returned - # otherwise, the parent node is queried. - flags_root: bool = False - - resolver_cache: Dict[str, Any] = field(default_factory=lambda: defaultdict(dict)) - - def __post_init__(self) -> None: - if self.flags is None: - self.flags = {} - - @property - def type_hint(self) -> Union[Type[Any], Any]: - """Compute `type_hint` from `self.optional` and `self.ref_type`""" - # For compatibility with pickled OmegaConf objects created using older - # versions of OmegaConf, we store `ref_type` and `object_type` - # separately (rather than storing `type_hint` directly). - if self.optional: - return Optional[self.ref_type] - else: - return self.ref_type - - -@dataclass -class ContainerMetadata(Metadata): - key_type: Any = None - element_type: Any = None - - def __post_init__(self) -> None: - if self.ref_type is None: - self.ref_type = Any - assert self.key_type is Any or isinstance(self.key_type, type) - if self.element_type is not None: - if not is_valid_value_annotation(self.element_type): - raise ValidationError( - f"Unsupported value type: '{type_str(self.element_type, include_module_name=True)}'" - ) - - if self.flags is None: - self.flags = {} - - -class Node(ABC): - _metadata: Metadata - - _parent: Optional["Box"] - _flags_cache: Optional[Dict[str, Optional[bool]]] - - def __init__(self, parent: Optional["Box"], metadata: Metadata): - self.__dict__["_metadata"] = metadata - self.__dict__["_parent"] = parent - self.__dict__["_flags_cache"] = None - - def __getstate__(self) -> Dict[str, Any]: - # Overridden to ensure that the flags cache is cleared on serialization. - state_dict = copy.copy(self.__dict__) - del state_dict["_flags_cache"] - return state_dict - - def __setstate__(self, state_dict: Dict[str, Any]) -> None: - self.__dict__.update(state_dict) - self.__dict__["_flags_cache"] = None - - def _set_parent(self, parent: Optional["Box"]) -> None: - assert parent is None or isinstance(parent, Box) - self.__dict__["_parent"] = parent - self._invalidate_flags_cache() - - def _invalidate_flags_cache(self) -> None: - self.__dict__["_flags_cache"] = None - - def _get_parent(self) -> Optional["Box"]: - parent = self.__dict__["_parent"] - assert parent is None or isinstance(parent, Box) - return parent - - def _get_parent_container(self) -> Optional["Container"]: - """ - Like _get_parent, but returns the grandparent - in the case where `self` is wrapped by a UnionNode. - """ - parent = self.__dict__["_parent"] - assert parent is None or isinstance(parent, Box) - - if isinstance(parent, UnionNode): - grandparent = parent.__dict__["_parent"] - assert grandparent is None or isinstance(grandparent, Container) - return grandparent - else: - assert parent is None or isinstance(parent, Container) - return parent - - def _set_flag( - self, - flags: Union[List[str], str], - values: Union[List[Optional[bool]], Optional[bool]], - ) -> "Node": - if isinstance(flags, str): - flags = [flags] - - if values is None or isinstance(values, bool): - values = [values] - - if len(values) == 1: - values = len(flags) * values - - if len(flags) != len(values): - raise ValueError("Inconsistent lengths of input flag names and values") - - for idx, flag in enumerate(flags): - value = values[idx] - if value is None: - assert self._metadata.flags is not None - if flag in self._metadata.flags: - del self._metadata.flags[flag] - else: - assert self._metadata.flags is not None - self._metadata.flags[flag] = value - self._invalidate_flags_cache() - return self - - def _get_node_flag(self, flag: str) -> Optional[bool]: - """ - :param flag: flag to inspect - :return: the state of the flag on this node. - """ - assert self._metadata.flags is not None - return self._metadata.flags.get(flag) - - def _get_flag(self, flag: str) -> Optional[bool]: - cache = self.__dict__["_flags_cache"] - if cache is None: - cache = self.__dict__["_flags_cache"] = {} - - ret = cache.get(flag, _DEFAULT_MARKER_) - if ret is _DEFAULT_MARKER_: - ret = self._get_flag_no_cache(flag) - cache[flag] = ret - assert ret is None or isinstance(ret, bool) - return ret - - def _get_flag_no_cache(self, flag: str) -> Optional[bool]: - """ - Returns True if this config node flag is set - A flag is set if node.set_flag(True) was called - or one if it's parents is flag is set - :return: - """ - flags = self._metadata.flags - assert flags is not None - if flag in flags and flags[flag] is not None: - return flags[flag] - - if self._is_flags_root(): - return None - - parent = self._get_parent() - if parent is None: - return None - else: - # noinspection PyProtectedMember - return parent._get_flag(flag) - - def _format_and_raise( - self, - key: Any, - value: Any, - cause: Exception, - msg: Optional[str] = None, - type_override: Any = None, - ) -> None: - format_and_raise( - node=self, - key=key, - value=value, - msg=str(cause) if msg is None else msg, - cause=cause, - type_override=type_override, - ) - assert False - - @abstractmethod - def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: - ... - - def _dereference_node(self) -> "Node": - node = self._dereference_node_impl(throw_on_resolution_failure=True) - assert node is not None - return node - - def _maybe_dereference_node( - self, - throw_on_resolution_failure: bool = False, - memo: Optional[Set[int]] = None, - ) -> Optional["Node"]: - return self._dereference_node_impl( - throw_on_resolution_failure=throw_on_resolution_failure, - memo=memo, - ) - - def _dereference_node_impl( - self, - throw_on_resolution_failure: bool, - memo: Optional[Set[int]] = None, - ) -> Optional["Node"]: - if not self._is_interpolation(): - return self - - parent = self._get_parent_container() - if parent is None: - if throw_on_resolution_failure: - raise InterpolationResolutionError("Cannot resolve interpolation for a node without a parent") - return None - assert parent is not None - key = self._key() - return parent._resolve_interpolation_from_parse_tree( - parent=parent, - key=key, - value=self, - parse_tree=parse(_get_value(self)), - throw_on_resolution_failure=throw_on_resolution_failure, - memo=memo, - ) - - def _get_root(self) -> "Container": - root: Optional[Box] = self._get_parent() - if root is None: - assert isinstance(self, Container) - return self - assert root is not None and isinstance(root, Box) - while root._get_parent() is not None: - root = root._get_parent() - assert root is not None and isinstance(root, Box) - assert root is not None and isinstance(root, Container) - return root - - def _is_missing(self) -> bool: - """ - Check if the node's value is `???` (does *not* resolve interpolations). - """ - return _is_missing_value(self) - - def _is_none(self) -> bool: - """ - Check if the node's value is `None` (does *not* resolve interpolations). - """ - return self._value() is None - - @abstractmethod - def __eq__(self, other: Any) -> bool: - ... - - @abstractmethod - def __ne__(self, other: Any) -> bool: - ... - - @abstractmethod - def __hash__(self) -> int: - ... - - @abstractmethod - def _value(self) -> Any: - ... - - @abstractmethod - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - ... - - @abstractmethod - def _is_optional(self) -> bool: - ... - - @abstractmethod - def _is_interpolation(self) -> bool: - ... - - def _key(self) -> Any: - return self._metadata.key - - def _set_key(self, key: Any) -> None: - self._metadata.key = key - - def _is_flags_root(self) -> bool: - return self._metadata.flags_root - - def _set_flags_root(self, flags_root: bool) -> None: - if self._metadata.flags_root != flags_root: - self._metadata.flags_root = flags_root - self._invalidate_flags_cache() - - def _has_ref_type(self) -> bool: - return self._metadata.ref_type is not Any - - -class Box(Node): - """ - Base class for nodes that can contain other nodes. - Concrete subclasses include DictConfig, ListConfig, and UnionNode. - """ - - _content: Any - - def __init__(self, parent: Optional["Box"], metadata: Metadata): - super().__init__(parent=parent, metadata=metadata) - self.__dict__["_content"] = None - - def __copy__(self) -> Any: - # real shallow copy is impossible because of the reference to the parent. - return copy.deepcopy(self) - - def _re_parent(self) -> None: - from .dictconfig import DictConfig - from .listconfig import ListConfig - - # update parents of first level Config nodes to self - - if isinstance(self, DictConfig): - content = self.__dict__["_content"] - if isinstance(content, dict): - for _key, value in self.__dict__["_content"].items(): - if value is not None: - value._set_parent(self) - if isinstance(value, Box): - value._re_parent() - elif isinstance(self, ListConfig): - content = self.__dict__["_content"] - if isinstance(content, list): - for item in self.__dict__["_content"]: - if item is not None: - item._set_parent(self) - if isinstance(item, Box): - item._re_parent() - elif isinstance(self, UnionNode): - content = self.__dict__["_content"] - if isinstance(content, Node): - content._set_parent(self) - if isinstance(content, Box): # pragma: no cover - # No coverage here as support for containers inside - # UnionNode is not yet implemented - content._re_parent() - - -class Container(Box): - """ - Container tagging interface - """ - - _metadata: ContainerMetadata - - @abstractmethod - def _get_child( - self, - key: Any, - validate_access: bool = True, - validate_key: bool = True, - throw_on_missing_value: bool = False, - throw_on_missing_key: bool = False, - ) -> Union[Optional[Node], List[Optional[Node]]]: - ... - - @abstractmethod - def _get_node( - self, - key: Any, - validate_access: bool = True, - validate_key: bool = True, - throw_on_missing_value: bool = False, - throw_on_missing_key: bool = False, - ) -> Union[Optional[Node], List[Optional[Node]]]: - ... - - @abstractmethod - def __delitem__(self, key: Any) -> None: - ... - - @abstractmethod - def __setitem__(self, key: Any, value: Any) -> None: - ... - - @abstractmethod - def __iter__(self) -> Iterator[Any]: - ... - - @abstractmethod - def __getitem__(self, key_or_index: Any) -> Any: - ... - - def _resolve_key_and_root(self, key: str) -> Tuple["Container", str]: - orig = key - if not key.startswith("."): - return self._get_root(), key - else: - root: Optional[Container] = self - assert key.startswith(".") - while True: - assert root is not None - key = key[1:] - if not key.startswith("."): - break - root = root._get_parent_container() - if root is None: - raise ConfigKeyError(f"Error resolving key '{orig}'") - - return root, key - - def _select_impl( - self, - key: str, - throw_on_missing: bool, - throw_on_resolution_failure: bool, - memo: Optional[Set[int]] = None, - ) -> Tuple[Optional["Container"], Optional[str], Optional[Node]]: - """ - Select a value using dot separated key sequence - """ - from .omegaconf import _select_one - - if key == "": - return self, "", self - - split = split_key(key) - root: Optional[Container] = self - for i in range(len(split) - 1): - if root is None: - break - - k = split[i] - ret, _ = _select_one( - c=root, - key=k, - throw_on_missing=throw_on_missing, - throw_on_type_error=throw_on_resolution_failure, - ) - if isinstance(ret, Node): - ret = ret._maybe_dereference_node( - throw_on_resolution_failure=throw_on_resolution_failure, - memo=memo, - ) - - if ret is not None and not isinstance(ret, Container): - parent_key = ".".join(split[0 : i + 1]) - child_key = split[i + 1] - raise ConfigTypeError( - f"Error trying to access {key}: node `{parent_key}` " - f"is not a container and thus cannot contain `{child_key}`" - ) - root = ret - - if root is None: - return None, None, None - - last_key = split[-1] - value, _ = _select_one( - c=root, - key=last_key, - throw_on_missing=throw_on_missing, - throw_on_type_error=throw_on_resolution_failure, - ) - if value is None: - return root, last_key, None - - if memo is not None: - vid = id(value) - if vid in memo: - raise InterpolationResolutionError("Recursive interpolation detected") - # push to memo "stack" - memo.add(vid) - - try: - value = root._maybe_resolve_interpolation( - parent=root, - key=last_key, - value=value, - throw_on_resolution_failure=throw_on_resolution_failure, - memo=memo, - ) - finally: - if memo is not None: - # pop from memo "stack" - memo.remove(vid) - - return root, last_key, value - - def _resolve_interpolation_from_parse_tree( - self, - parent: Optional["Container"], - value: "Node", - key: Any, - parse_tree: OmegaConfGrammarParser.ConfigValueContext, - throw_on_resolution_failure: bool, - memo: Optional[Set[int]], - ) -> Optional["Node"]: - """ - Resolve an interpolation. - - This happens in two steps: - 1. The parse tree is visited, which outputs either a `Node` (e.g., - for node interpolations "${foo}"), a string (e.g., for string - interpolations "hello ${name}", or any other arbitrary value - (e.g., or custom interpolations "${foo:bar}"). - 2. This output is potentially validated and converted when the node - being resolved (`value`) is typed. - - If an error occurs in one of the above steps, an `InterpolationResolutionError` - (or a subclass of it) is raised, *unless* `throw_on_resolution_failure` is set - to `False` (in which case the return value is `None`). - - :param parent: Parent of the node being resolved. - :param value: Node being resolved. - :param key: The associated key in the parent. - :param parse_tree: The parse tree as obtained from `grammar_parser.parse()`. - :param throw_on_resolution_failure: If `False`, then exceptions raised during - the resolution of the interpolation are silenced, and instead `None` is - returned. - - :return: A `Node` that contains the interpolation result. This may be an existing - node in the config (in the case of a node interpolation "${foo}"), or a new - node that is created to wrap the interpolated value. It is `None` if and only if - `throw_on_resolution_failure` is `False` and an error occurs during resolution. - """ - - try: - resolved = self.resolve_parse_tree(parse_tree=parse_tree, node=value, key=key, memo=memo) - except InterpolationResolutionError: - if throw_on_resolution_failure: - raise - return None - - return self._validate_and_convert_interpolation_result( - parent=parent, - value=value, - key=key, - resolved=resolved, - throw_on_resolution_failure=throw_on_resolution_failure, - ) - - def _validate_and_convert_interpolation_result( - self, - parent: Optional["Container"], - value: "Node", - key: Any, - resolved: Any, - throw_on_resolution_failure: bool, - ) -> Optional["Node"]: - from .nodes import AnyNode, InterpolationResultNode, ValueNode - - # If the output is not a Node already (e.g., because it is the output of a - # custom resolver), then we will need to wrap it within a Node. - must_wrap = not isinstance(resolved, Node) - - # If the node is typed, validate (and possibly convert) the result. - if isinstance(value, ValueNode) and not isinstance(value, AnyNode): - res_value = _get_value(resolved) - try: - conv_value = value.validate_and_convert(res_value) - except ValidationError as e: - if throw_on_resolution_failure: - self._format_and_raise( - key=key, - value=res_value, - cause=e, - msg=f"While dereferencing interpolation '{value}': {e}", - type_override=InterpolationValidationError, - ) - return None - - # If the converted value is of the same type, it means that no conversion - # was actually needed. As a result, we can keep the original `resolved` - # (and otherwise, the converted value must be wrapped into a new node). - if type(conv_value) != type(res_value): - must_wrap = True - resolved = conv_value - - if must_wrap: - return InterpolationResultNode(value=resolved, key=key, parent=parent) - else: - assert isinstance(resolved, Node) - return resolved - - def _validate_not_dereferencing_to_parent(self, node: Node, target: Node) -> None: - parent: Optional[Node] = node - while parent is not None: - if parent is target: - raise InterpolationResolutionError("Interpolation to parent node detected") - parent = parent._get_parent() - - def _resolve_node_interpolation(self, inter_key: str, memo: Optional[Set[int]]) -> "Node": - """A node interpolation is of the form `${foo.bar}`""" - try: - root_node, inter_key = self._resolve_key_and_root(inter_key) - except ConfigKeyError as exc: - raise InterpolationKeyError( - f"ConfigKeyError while resolving interpolation: {exc}" - ).with_traceback(sys.exc_info()[2]) - - try: - parent, last_key, value = root_node._select_impl( - inter_key, - throw_on_missing=True, - throw_on_resolution_failure=True, - memo=memo, - ) - except MissingMandatoryValue as exc: - raise InterpolationToMissingValueError( - f"MissingMandatoryValue while resolving interpolation: {exc}" - ).with_traceback(sys.exc_info()[2]) - - if parent is None or value is None: - raise InterpolationKeyError(f"Interpolation key '{inter_key}' not found") - else: - self._validate_not_dereferencing_to_parent(node=self, target=value) - return value - - def _evaluate_custom_resolver( - self, - key: Any, - node: Node, - inter_type: str, - inter_args: Tuple[Any, ...], - inter_args_str: Tuple[str, ...], - ) -> Any: - from omegaconf import OmegaConf - - resolver = OmegaConf._get_resolver(inter_type) - if resolver is not None: - root_node = self._get_root() - return resolver( - root_node, - self, - node, - inter_args, - inter_args_str, - ) - else: - raise UnsupportedInterpolationType(f"Unsupported interpolation type {inter_type}") - - def _maybe_resolve_interpolation( - self, - parent: Optional["Container"], - key: Any, - value: Node, - throw_on_resolution_failure: bool, - memo: Optional[Set[int]] = None, - ) -> Optional[Node]: - value_kind = get_value_kind(value) - if value_kind != ValueKind.INTERPOLATION: - return value - - parse_tree = parse(_get_value(value)) - return self._resolve_interpolation_from_parse_tree( - parent=parent, - value=value, - key=key, - parse_tree=parse_tree, - throw_on_resolution_failure=throw_on_resolution_failure, - memo=memo if memo is not None else set(), - ) - - def resolve_parse_tree( - self, - parse_tree: ParserRuleContext, - node: Node, - memo: Optional[Set[int]] = None, - key: Optional[Any] = None, - ) -> Any: - """ - Resolve a given parse tree into its value. - - We make no assumption here on the type of the tree's root, so that the - return value may be of any type. - """ - - def node_interpolation_callback(inter_key: str, memo: Optional[Set[int]]) -> Optional["Node"]: - return self._resolve_node_interpolation(inter_key=inter_key, memo=memo) - - def resolver_interpolation_callback( - name: str, args: Tuple[Any, ...], args_str: Tuple[str, ...] - ) -> Any: - return self._evaluate_custom_resolver( - key=key, - node=node, - inter_type=name, - inter_args=args, - inter_args_str=args_str, - ) - - visitor = GrammarVisitor( - node_interpolation_callback=node_interpolation_callback, - resolver_interpolation_callback=resolver_interpolation_callback, - memo=memo, - ) - try: - return visitor.visit(parse_tree) - except InterpolationResolutionError: - raise - except Exception as exc: - # Other kinds of exceptions are wrapped in an `InterpolationResolutionError`. - raise InterpolationResolutionError( - f"{type(exc).__name__} raised while resolving interpolation: {exc}" - ).with_traceback(sys.exc_info()[2]) - - def _invalidate_flags_cache(self) -> None: - from .dictconfig import DictConfig - from .listconfig import ListConfig - - # invalidate subtree cache only if the cache is initialized in this node. - - if self.__dict__["_flags_cache"] is not None: - self.__dict__["_flags_cache"] = None - if isinstance(self, DictConfig): - content = self.__dict__["_content"] - if isinstance(content, dict): - for value in self.__dict__["_content"].values(): - value._invalidate_flags_cache() - elif isinstance(self, ListConfig): - content = self.__dict__["_content"] - if isinstance(content, list): - for item in self.__dict__["_content"]: - item._invalidate_flags_cache() - - -class SCMode(Enum): - DICT = 1 # Convert to plain dict - DICT_CONFIG = 2 # Keep as OmegaConf DictConfig - INSTANTIATE = 3 # Create a dataclass or attrs class instance - - -class UnionNode(Box): - """ - This class handles Union type hints. The `_content` attribute is either a - child node that is compatible with the given Union ref_type, or it is a - special value (None or MISSING or interpolation). - - Much of the logic for e.g. value assignment and type validation is - delegated to the child node. As such, UnionNode functions as a - "pass-through" node. User apps and downstream libraries should not need to - know about UnionNode (assuming they only use OmegaConf's public API). - """ - - _parent: Optional[Container] - _content: Union[Node, None, str] - - def __init__( - self, - content: Any, - ref_type: Any, - is_optional: bool = True, - key: Any = None, - parent: Optional[Box] = None, - ) -> None: - try: - if not is_union_annotation(ref_type): # pragma: no cover - msg = ( - f"UnionNode got unexpected ref_type {ref_type}. Please file a bug" - + " report at https://github.com/omry/omegaconf/issues" - ) - raise AssertionError(msg) - if not isinstance(parent, (Container, NoneType)): - raise ConfigTypeError("Parent type is not omegaconf.Container") - super().__init__( - parent=parent, - metadata=Metadata( - ref_type=ref_type, - object_type=None, - optional=is_optional, - key=key, - flags={"convert": False}, - ), - ) - self._set_value(content) - except Exception as ex: - format_and_raise(node=None, key=key, value=content, msg=str(ex), cause=ex) - - def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: - parent = self._get_parent() - if parent is None: - if self._metadata.key is None: - return "" - else: - return str(self._metadata.key) - else: - return parent._get_full_key(self._metadata.key) - - def __eq__(self, other: Any) -> bool: - content = self.__dict__["_content"] - if isinstance(content, Node): - ret = content.__eq__(other) - elif isinstance(other, Node): - ret = other.__eq__(content) - else: - ret = content.__eq__(other) - assert isinstance(ret, (bool, type(NotImplemented))) - return ret - - def __ne__(self, other: Any) -> bool: - x = self.__eq__(other) - if x is NotImplemented: - return NotImplemented - return not x - - def __hash__(self) -> int: - return hash(self.__dict__["_content"]) - - def _value(self) -> Union[Node, None, str]: - content = self.__dict__["_content"] - assert isinstance(content, (Node, NoneType, str)) - return content - - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - previous_content = self.__dict__["_content"] - previous_metadata = self.__dict__["_metadata"] - try: - self._set_value_impl(value, flags) - except Exception as e: - self.__dict__["_content"] = previous_content - self.__dict__["_metadata"] = previous_metadata - raise e - - def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - from omegaconf.omegaconf import _node_wrap - - ref_type = self._metadata.ref_type - type_hint = self._metadata.type_hint - - value = _get_value(value) - if _is_special(value): - assert isinstance(value, (str, NoneType)) - if value is None: - if not self._is_optional(): - raise ValidationError( - f"Value '$VALUE' is incompatible with type hint '{type_str(type_hint)}'" - ) - self.__dict__["_content"] = value - elif isinstance(value, Container): - raise ValidationError( - f"Cannot assign container '$VALUE' of type '$VALUE_TYPE' to {type_str(type_hint)}" - ) - else: - for candidate_ref_type in ref_type.__args__: - try: - self.__dict__["_content"] = _node_wrap( - value=value, - ref_type=candidate_ref_type, - is_optional=False, - key=None, - parent=self, - ) - break - except ValidationError: - continue - else: - raise ValidationError( - f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_str(type_hint)}'" - ) - - def _is_optional(self) -> bool: - return self.__dict__["_metadata"].optional is True - - def _is_interpolation(self) -> bool: - return _is_interpolation(self.__dict__["_content"]) - - def __str__(self) -> str: - return str(self.__dict__["_content"]) - - def __repr__(self) -> str: - return repr(self.__dict__["_content"]) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "UnionNode": - res = object.__new__(type(self)) - for key, value in self.__dict__.items(): - if key not in ("_content", "_parent"): - res.__dict__[key] = copy.deepcopy(value, memo=memo) - - src_content = self.__dict__["_content"] - if isinstance(src_content, Node): - old_parent = src_content.__dict__["_parent"] - try: - src_content.__dict__["_parent"] = None - content_copy = copy.deepcopy(src_content, memo=memo) - content_copy.__dict__["_parent"] = res - finally: - src_content.__dict__["_parent"] = old_parent - else: - # None and strings can be assigned as is - content_copy = src_content - - res.__dict__["_content"] = content_copy - res.__dict__["_parent"] = self.__dict__["_parent"] - return res diff --git a/omegaconf/basecontainer.py b/omegaconf/basecontainer.py deleted file mode 100644 index c22cbfd1d..000000000 --- a/omegaconf/basecontainer.py +++ /dev/null @@ -1,916 +0,0 @@ -import copy -import sys -from abc import ABC, abstractmethod -from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union - -import yaml - -from ._utils import ( - _DEFAULT_MARKER_, - ValueKind, - _ensure_container, - _get_value, - _is_interpolation, - _is_missing_value, - _is_none, - _is_special, - _resolve_optional, - get_structured_config_data, - get_type_hint, - get_value_kind, - get_yaml_loader, - is_container_annotation, - is_dict_annotation, - is_list_annotation, - is_primitive_dict, - is_primitive_type_annotation, - is_structured_config, - is_tuple_annotation, - is_union_annotation, -) -from .base import ( - Box, - Container, - ContainerMetadata, - DictKeyType, - Node, - SCMode, - UnionNode, -) -from .errors import ( - ConfigCycleDetectedException, - ConfigTypeError, - InterpolationResolutionError, - KeyValidationError, - MissingMandatoryValue, - OmegaConfBaseException, - ReadonlyConfigError, - ValidationError, -) - -if TYPE_CHECKING: - from .dictconfig import DictConfig # pragma: no cover - - -class BaseContainer(Container, ABC): - _resolvers: ClassVar[Dict[str, Any]] = {} - - def __init__(self, parent: Optional[Box], metadata: ContainerMetadata): - if not (parent is None or isinstance(parent, Box)): - raise ConfigTypeError("Parent type is not omegaconf.Box") - super().__init__(parent=parent, metadata=metadata) - - def _get_child( - self, - key: Any, - validate_access: bool = True, - validate_key: bool = True, - throw_on_missing_value: bool = False, - throw_on_missing_key: bool = False, - ) -> Union[Optional[Node], List[Optional[Node]]]: - """Like _get_node, passing through to the nearest concrete Node.""" - child = self._get_node( - key=key, - validate_access=validate_access, - validate_key=validate_key, - throw_on_missing_value=throw_on_missing_value, - throw_on_missing_key=throw_on_missing_key, - ) - if isinstance(child, UnionNode) and not _is_special(child): - value = child._value() - assert isinstance(value, Node) and not isinstance(value, UnionNode) - child = value - return child - - def _resolve_with_default( - self, - key: Union[DictKeyType, int], - value: Node, - default_value: Any = _DEFAULT_MARKER_, - ) -> Any: - """returns the value with the specified key, like obj.key and obj['key']""" - if _is_missing_value(value): - if default_value is not _DEFAULT_MARKER_: - return default_value - raise MissingMandatoryValue("Missing mandatory value: $FULL_KEY") - - resolved_node = self._maybe_resolve_interpolation( - parent=self, - key=key, - value=value, - throw_on_resolution_failure=True, - ) - - return _get_value(resolved_node) - - def __str__(self) -> str: - return self.__repr__() - - def __repr__(self) -> str: - if self.__dict__["_content"] is None: - return "None" - elif self._is_interpolation() or self._is_missing(): - v = self.__dict__["_content"] - return f"'{v}'" - else: - return self.__dict__["_content"].__repr__() # type: ignore - - # Support pickle - def __getstate__(self) -> Dict[str, Any]: - dict_copy = copy.copy(self.__dict__) - - # no need to serialize the flags cache, it can be re-constructed later - dict_copy.pop("_flags_cache", None) - - dict_copy["_metadata"] = copy.copy(dict_copy["_metadata"]) - ref_type = self._metadata.ref_type - if is_container_annotation(ref_type): - if is_dict_annotation(ref_type): - dict_copy["_metadata"].ref_type = Dict - elif is_list_annotation(ref_type): - dict_copy["_metadata"].ref_type = List - else: - assert False - if sys.version_info < (3, 7): # pragma: no cover - element_type = self._metadata.element_type - if is_union_annotation(element_type): - raise OmegaConfBaseException( - "Serializing structured configs with `Union` element type requires python >= 3.7" - ) - return dict_copy - - # Support pickle - def __setstate__(self, d: Dict[str, Any]) -> None: - from omegaconf import DictConfig - from omegaconf._utils import is_generic_dict, is_generic_list - - if isinstance(self, DictConfig): - key_type = d["_metadata"].key_type - - # backward compatibility to load OmegaConf 2.0 configs - if key_type is None: - key_type = Any - d["_metadata"].key_type = key_type - - element_type = d["_metadata"].element_type - - # backward compatibility to load OmegaConf 2.0 configs - if element_type is None: - element_type = Any - d["_metadata"].element_type = element_type - - ref_type = d["_metadata"].ref_type - if is_container_annotation(ref_type): - if is_generic_dict(ref_type): - d["_metadata"].ref_type = Dict[key_type, element_type] # type: ignore - elif is_generic_list(ref_type): - d["_metadata"].ref_type = List[element_type] # type: ignore - else: - assert False - - d["_flags_cache"] = None - self.__dict__.update(d) - - @abstractmethod - def __delitem__(self, key: Any) -> None: - ... - - def __len__(self) -> int: - if self._is_none() or self._is_missing() or self._is_interpolation(): - return 0 - content = self.__dict__["_content"] - return len(content) - - def merge_with_cli(self) -> None: - args_list = sys.argv[1:] - self.merge_with_dotlist(args_list) - - def merge_with_dotlist(self, dotlist: List[str]) -> None: - from omegaconf import OmegaConf - - def fail() -> None: - raise ValueError("Input list must be a list or a tuple of strings") - - if not isinstance(dotlist, (list, tuple)): - fail() - - for arg in dotlist: - if not isinstance(arg, str): - fail() - - idx = arg.find("=") - if idx == -1: - key = arg - value = None - else: - key = arg[0:idx] - value = arg[idx + 1 :] - value = yaml.load(value, Loader=get_yaml_loader()) - - OmegaConf.update(self, key, value) - - def is_empty(self) -> bool: - """return true if config is empty""" - return len(self.__dict__["_content"]) == 0 - - @staticmethod - def _to_content( - conf: Container, - resolve: bool, - throw_on_missing: bool, - enum_to_str: bool = False, - structured_config_mode: SCMode = SCMode.DICT, - ) -> Union[None, Any, str, Dict[DictKeyType, Any], List[Any]]: - from omegaconf import MISSING, DictConfig, ListConfig - - def convert(val: Node) -> Any: - value = val._value() - if enum_to_str and isinstance(value, Enum): - value = f"{value.name}" - - return value - - def get_node_value(key: Union[DictKeyType, int]) -> Any: - try: - node = conf._get_child(key, throw_on_missing_value=throw_on_missing) - except MissingMandatoryValue as e: - conf._format_and_raise(key=key, value=None, cause=e) - assert isinstance(node, Node) - if resolve: - try: - node = node._dereference_node() - except InterpolationResolutionError as e: - conf._format_and_raise(key=key, value=None, cause=e) - - if isinstance(node, Container): - value = BaseContainer._to_content( - node, - resolve=resolve, - throw_on_missing=throw_on_missing, - enum_to_str=enum_to_str, - structured_config_mode=structured_config_mode, - ) - else: - value = convert(node) - return value - - if conf._is_none(): - return None - elif conf._is_missing(): - if throw_on_missing: - conf._format_and_raise( - key=None, - value=None, - cause=MissingMandatoryValue("Missing mandatory value"), - ) - else: - return MISSING - elif not resolve and conf._is_interpolation(): - inter = conf._value() - assert isinstance(inter, str) - return inter - - if resolve: - _conf = conf._dereference_node() - assert isinstance(_conf, Container) - conf = _conf - - if isinstance(conf, DictConfig): - if ( - conf._metadata.object_type not in (dict, None) - and structured_config_mode == SCMode.DICT_CONFIG - ): - return conf - if structured_config_mode == SCMode.INSTANTIATE and is_structured_config( - conf._metadata.object_type - ): - return conf._to_object() - - retdict: Dict[DictKeyType, Any] = {} - for key in conf.keys(): - value = get_node_value(key) - if enum_to_str and isinstance(key, Enum): - key = f"{key.name}" - retdict[key] = value - return retdict - elif isinstance(conf, ListConfig): - retlist: List[Any] = [] - for index in range(len(conf)): - item = get_node_value(index) - retlist.append(item) - - return retlist - assert False - - @staticmethod - def _map_merge( - dest: "BaseContainer", - src: "BaseContainer", - extend_lists: bool = False, - remove_duplicates: bool = False, - ) -> None: - """merge src into dest and return a new copy, does not modified input""" - from omegaconf import AnyNode, DictConfig, ValueNode - - assert isinstance(dest, DictConfig) - assert isinstance(src, DictConfig) - src_type = src._metadata.object_type - src_ref_type = get_type_hint(src) - assert src_ref_type is not None - - # If source DictConfig is: - # - None => set the destination DictConfig to None - # - an interpolation => set the destination DictConfig to be the same interpolation - if src._is_none() or src._is_interpolation(): - dest._set_value(src._value()) - _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) - return - - dest._validate_merge(value=src) - - def expand(node: Container) -> None: - rt = node._metadata.ref_type - val: Any - if rt is not Any: - if is_dict_annotation(rt): - val = {} - elif is_list_annotation(rt) or is_tuple_annotation(rt): - val = [] - else: - val = rt - elif isinstance(node, DictConfig): - val = {} - else: - assert False - - node._set_value(val) - - if src._is_missing() and not dest._is_missing() and is_structured_config(src_ref_type): - # Replace `src` with a prototype of its corresponding structured config - # whose fields are all missing (to avoid overwriting fields in `dest`). - assert src_type is None # src missing, so src's object_type should be None - src_type = src_ref_type - src = _create_structured_with_missing_fields(ref_type=src_ref_type, object_type=src_type) - - if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing(): - expand(dest) - - src_items = list(src) if not src._is_missing() else [] - for key in src_items: - src_node = src._get_node(key, validate_access=False) - dest_node = dest._get_node(key, validate_access=False) - assert isinstance(src_node, Node) - assert dest_node is None or isinstance(dest_node, Node) - src_value = _get_value(src_node) - - src_vk = get_value_kind(src_node) - src_node_missing = src_vk is ValueKind.MANDATORY_MISSING - - if isinstance(dest_node, DictConfig): - dest_node._validate_merge(value=src_node) - - if ( - isinstance(dest_node, Container) - and dest_node._is_none() - and not src_node_missing - and not _is_none(src_node, resolve=True) - ): - expand(dest_node) - - if dest_node is not None and dest_node._is_interpolation(): - target_node = dest_node._maybe_dereference_node() - if isinstance(target_node, Container): - dest[key] = target_node - dest_node = dest._get_node(key) - - is_optional, et = _resolve_optional(dest._metadata.element_type) - if dest_node is None and is_structured_config(et) and not src_node_missing: - # merging into a new node. Use element_type as a base - dest[key] = DictConfig(et, parent=dest, ref_type=et, is_optional=is_optional) - dest_node = dest._get_node(key) - - if dest_node is not None: - if isinstance(dest_node, BaseContainer): - if isinstance(src_node, BaseContainer): - dest_node._merge_with( - src_node, - extend_lists=extend_lists, - remove_duplicates=remove_duplicates, - ) - elif not src_node_missing: - dest.__setitem__(key, src_node) - else: - if isinstance(src_node, BaseContainer): - dest.__setitem__(key, src_node) - else: - assert isinstance(dest_node, (ValueNode, UnionNode)) - assert isinstance(src_node, (ValueNode, UnionNode)) - try: - if isinstance(dest_node, AnyNode): - if src_node_missing: - node = copy.copy(src_node) - # if src node is missing, use the value from the dest_node, - # but validate it against the type of the src node before assigment - node._set_value(dest_node._value()) - else: - node = src_node - dest.__setitem__(key, node) - else: - if not src_node_missing: - dest_node._set_value(src_value) - - except (ValidationError, ReadonlyConfigError) as e: - dest._format_and_raise(key=key, value=src_value, cause=e) - else: - from omegaconf import open_dict - - if is_structured_config(src_type): - # verified to be compatible above in _validate_merge - with open_dict(dest): - dest[key] = src._get_node(key) - else: - dest[key] = src._get_node(key) - - _update_types(node=dest, ref_type=src_ref_type, object_type=src_type) - - # explicit flags on the source config are replacing the flag values in the destination - flags = src._metadata.flags - assert flags is not None - for flag, value in flags.items(): - if value is not None: - dest._set_flag(flag, value) - - @staticmethod - def _list_merge(dest: Any, src: Any, extend_lists: bool = False, remove_duplicates: bool = False) -> None: - from omegaconf import DictConfig, ListConfig, OmegaConf - - assert isinstance(dest, ListConfig) - assert isinstance(src, ListConfig) - - if src._is_none(): - dest._set_value(None) - elif src._is_missing(): - # do not change dest if src is MISSING. - if dest._metadata.element_type is Any: - dest._metadata.element_type = src._metadata.element_type - elif src._is_interpolation(): - dest._set_value(src._value()) - else: - temp_target = ListConfig(content=[], parent=dest._get_parent()) - temp_target.__dict__["_metadata"] = copy.deepcopy(dest.__dict__["_metadata"]) - is_optional, et = _resolve_optional(dest._metadata.element_type) - if is_structured_config(et): - prototype = DictConfig(et, ref_type=et, is_optional=is_optional) - for item in src._iter_ex(resolve=False): - if isinstance(item, DictConfig): - item = OmegaConf.merge(prototype, item) - temp_target.append(item) - else: - for item in src._iter_ex(resolve=False): - temp_target.append(item) - - if extend_lists: - if remove_duplicates: - for entry in temp_target.__dict__["_content"]: - if entry not in dest.__dict__["_content"]: - dest.__dict__["_content"].append(entry) - else: - dest.__dict__["_content"].extend(temp_target.__dict__["_content"]) - else: - dest.__dict__["_content"] = temp_target.__dict__["_content"] - - # explicit flags on the source config are replacing the flag values in the destination - flags = src._metadata.flags - assert flags is not None - for flag, value in flags.items(): - if value is not None: - dest._set_flag(flag, value) - - def merge_with( - self, - *others: Union["BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any], - extend_lists: bool = False, - remove_duplicates: bool = False, - ) -> None: - try: - self._merge_with(*others, extend_lists=extend_lists, remove_duplicates=remove_duplicates) - except Exception as e: - self._format_and_raise(key=None, value=None, cause=e) - - def _merge_with( - self, - *others: Union["BaseContainer", Dict[str, Any], List[Any], Tuple[Any, ...], Any], - extend_lists: bool = False, - remove_duplicates: bool = False, - ) -> None: - from .dictconfig import DictConfig - from .listconfig import ListConfig - - """merge a list of other Config objects into this one, overriding as needed""" - for other in others: - if other is None: - raise ValueError("Cannot merge with a None config") - - my_flags = {} - if self._get_flag("allow_objects") is True: - my_flags = {"allow_objects": True} - other = _ensure_container(other, flags=my_flags) - - if isinstance(self, DictConfig) and isinstance(other, DictConfig): - BaseContainer._map_merge( - self, - other, - extend_lists=extend_lists, - remove_duplicates=remove_duplicates, - ) - elif isinstance(self, ListConfig) and isinstance(other, ListConfig): - BaseContainer._list_merge( - self, - other, - extend_lists=extend_lists, - remove_duplicates=remove_duplicates, - ) - else: - raise TypeError("Cannot merge DictConfig with ListConfig") - - # recursively correct the parent hierarchy after the merge - self._re_parent() - - # noinspection PyProtectedMember - def _set_item_impl(self, key: Any, value: Any) -> None: - """ - Changes the value of the node key with the desired value. If the node key doesn't - exist it creates a new one. - """ - from .nodes import AnyNode, ValueNode - - if isinstance(value, Node): - do_deepcopy = not self._get_flag("no_deepcopy_set_nodes") - if not do_deepcopy and isinstance(value, Box): - # if value is from the same config, perform a deepcopy no matter what. - if self._get_root() is value._get_root(): - do_deepcopy = True - - if do_deepcopy: - value = copy.deepcopy(value) - value._set_parent(None) - - try: - old = value._key() - value._set_key(key) - self._validate_set(key, value) - finally: - value._set_key(old) - else: - self._validate_set(key, value) - - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot change read-only config container") - - input_is_node = isinstance(value, Node) - target_node_ref = self._get_node(key) - assert target_node_ref is None or isinstance(target_node_ref, Node) - - input_is_typed_vnode = isinstance(value, ValueNode) and not isinstance(value, AnyNode) - - def get_target_type_hint(val: Any) -> Any: - if not is_structured_config(val): - type_hint = self._metadata.element_type - else: - target = self._get_node(key) - if target is None: - type_hint = self._metadata.element_type - else: - assert isinstance(target, Node) - type_hint = target._metadata.type_hint - return type_hint - - target_type_hint = get_target_type_hint(value) - _, target_ref_type = _resolve_optional(target_type_hint) - - def assign(value_key: Any, val: Node) -> None: - assert val._get_parent() is None - v = val - v._set_parent(self) - v._set_key(value_key) - _deep_update_type_hint(node=v, type_hint=self._metadata.element_type) - self.__dict__["_content"][value_key] = v - - if input_is_typed_vnode and not is_union_annotation(target_ref_type): - assign(key, value) - else: - # input is not a ValueNode, can be primitive or box - - special_value = _is_special(value) - # We use the `Node._set_value` method if the target node exists and: - # 1. the target has an explicit ref_type, or - # 2. the target is an AnyNode and the input is a primitive type. - should_set_value = target_node_ref is not None and ( - target_node_ref._has_ref_type() - or (isinstance(target_node_ref, AnyNode) and is_primitive_type_annotation(value)) - ) - if should_set_value: - if special_value and isinstance(value, Node): - value = value._value() - self.__dict__["_content"][key]._set_value(value) - elif input_is_node: - if ( - special_value - and (is_container_annotation(target_ref_type) or is_structured_config(target_ref_type)) - or is_primitive_type_annotation(target_ref_type) - or is_union_annotation(target_ref_type) - ): - value = _get_value(value) - self._wrap_value_and_set(key, value, target_type_hint) - else: - assign(key, value) - else: - self._wrap_value_and_set(key, value, target_type_hint) - - def _wrap_value_and_set(self, key: Any, val: Any, type_hint: Any) -> None: - from omegaconf.omegaconf import _maybe_wrap - - is_optional, ref_type = _resolve_optional(type_hint) - - try: - wrapped = _maybe_wrap( - ref_type=ref_type, - key=key, - value=val, - is_optional=is_optional, - parent=self, - ) - except ValidationError as e: - self._format_and_raise(key=key, value=val, cause=e) - self.__dict__["_content"][key] = wrapped - - @staticmethod - def _item_eq( - c1: Container, - k1: Union[DictKeyType, int], - c2: Container, - k2: Union[DictKeyType, int], - ) -> bool: - v1 = c1._get_child(k1) - v2 = c2._get_child(k2) - assert v1 is not None and v2 is not None - - assert isinstance(v1, Node) - assert isinstance(v2, Node) - - if v1._is_none() and v2._is_none(): - return True - - if v1._is_missing() and v2._is_missing(): - return True - - v1_inter = v1._is_interpolation() - v2_inter = v2._is_interpolation() - dv1: Optional[Node] = v1 - dv2: Optional[Node] = v2 - - if v1_inter: - dv1 = v1._maybe_dereference_node() - if v2_inter: - dv2 = v2._maybe_dereference_node() - - if v1_inter and v2_inter: - if dv1 is None or dv2 is None: - return v1 == v2 - else: - # both are not none, if both are containers compare as container - if isinstance(dv1, Container) and isinstance(dv2, Container): - if dv1 != dv2: - return False - dv1 = _get_value(dv1) - dv2 = _get_value(dv2) - return dv1 == dv2 - elif not v1_inter and not v2_inter: - v1 = _get_value(v1) - v2 = _get_value(v2) - ret = v1 == v2 - assert isinstance(ret, bool) - return ret - else: - dv1 = _get_value(dv1) - dv2 = _get_value(dv2) - ret = dv1 == dv2 - assert isinstance(ret, bool) - return ret - - def _is_optional(self) -> bool: - return self.__dict__["_metadata"].optional is True - - def _is_interpolation(self) -> bool: - return _is_interpolation(self.__dict__["_content"]) - - @abstractmethod - def _validate_get(self, key: Any, value: Any = None) -> None: - ... - - @abstractmethod - def _validate_set(self, key: Any, value: Any) -> None: - ... - - def _value(self) -> Any: - return self.__dict__["_content"] - - def _get_full_key(self, key: Union[DictKeyType, int, slice, None]) -> str: - from .listconfig import ListConfig - from .omegaconf import _select_one - - if not isinstance(key, (int, str, Enum, float, bool, slice, bytes, type(None))): - return "" - - def _slice_to_str(x: slice) -> str: - if x.step is not None: - return f"{x.start}:{x.stop}:{x.step}" - else: - return f"{x.start}:{x.stop}" - - def prepand( - full_key: str, - parent_type: Any, - cur_type: Any, - key: Optional[Union[DictKeyType, int, slice]], - ) -> str: - if key is None: - return full_key - - if isinstance(key, slice): - key = _slice_to_str(key) - elif isinstance(key, Enum): - key = key.name - else: - key = str(key) - - assert isinstance(key, str) - - if issubclass(parent_type, ListConfig): - if full_key != "": - if issubclass(cur_type, ListConfig): - full_key = f"[{key}]{full_key}" - else: - full_key = f"[{key}].{full_key}" - else: - full_key = f"[{key}]" - else: - if full_key == "": - full_key = key - else: - if issubclass(cur_type, ListConfig): - full_key = f"{key}{full_key}" - else: - full_key = f"{key}.{full_key}" - return full_key - - if key is not None and key != "": - assert isinstance(self, Container) - cur, _ = _select_one(c=self, key=str(key), throw_on_missing=False, throw_on_type_error=False) - if cur is None: - cur = self - full_key = prepand("", type(cur), None, key) - if cur._key() is not None: - full_key = prepand(full_key, type(cur._get_parent()), type(cur), cur._key()) - else: - full_key = prepand("", type(cur._get_parent()), type(cur), cur._key()) - else: - cur = self - if cur._key() is None: - return "" - full_key = self._key() - - assert cur is not None - memo = {id(cur)} # remember already visited nodes so as to detect cycles - while cur._get_parent() is not None: - cur = cur._get_parent() - if id(cur) in memo: - raise ConfigCycleDetectedException(f"Cycle when iterating over parents of key `{key!s}`") - memo.add(id(cur)) - assert cur is not None - if cur._key() is not None: - full_key = prepand(full_key, type(cur._get_parent()), type(cur), cur._key()) - - return full_key - - -def _create_structured_with_missing_fields( - ref_type: type, object_type: Optional[type] = None -) -> "DictConfig": - from . import MISSING, DictConfig - - cfg_data = get_structured_config_data(ref_type) - for v in cfg_data.values(): - v._set_value(MISSING) - - cfg = DictConfig(cfg_data) - cfg._metadata.optional, cfg._metadata.ref_type = _resolve_optional(ref_type) - cfg._metadata.object_type = object_type - - return cfg - - -def _update_types(node: Node, ref_type: Any, object_type: Optional[type]) -> None: - if object_type is not None and not is_primitive_dict(object_type): - node._metadata.object_type = object_type - - if node._metadata.ref_type is Any: - _deep_update_type_hint(node, ref_type) - - -def _deep_update_type_hint(node: Node, type_hint: Any) -> None: - """Ensure node is compatible with type_hint, mutating if necessary.""" - from omegaconf import DictConfig, ListConfig - - from ._utils import get_dict_key_value_types, get_list_element_type - - if type_hint is Any: - return - - _shallow_validate_type_hint(node, type_hint) - - new_is_optional, new_ref_type = _resolve_optional(type_hint) - node._metadata.ref_type = new_ref_type - node._metadata.optional = new_is_optional - - if is_list_annotation(new_ref_type) and isinstance(node, ListConfig): - new_element_type = get_list_element_type(new_ref_type) - node._metadata.element_type = new_element_type - if not _is_special(node): - for i in range(len(node)): - _deep_update_subnode(node, i, new_element_type) - - if is_dict_annotation(new_ref_type) and isinstance(node, DictConfig): - new_key_type, new_element_type = get_dict_key_value_types(new_ref_type) - node._metadata.key_type = new_key_type - node._metadata.element_type = new_element_type - if not _is_special(node): - for key in node: - if new_key_type is not Any and not isinstance(key, new_key_type): - raise KeyValidationError( - f"Key {key!r} ({type(key).__name__}) is incompatible" - + f" with key type hint '{new_key_type.__name__}'" - ) - _deep_update_subnode(node, key, new_element_type) - - -def _deep_update_subnode(node: BaseContainer, key: Any, value_type_hint: Any) -> None: - """Get node[key] and ensure it is compatible with value_type_hint, mutating if necessary.""" - subnode = node._get_node(key) - assert isinstance(subnode, Node) - if _is_special(subnode): - # Ensure special values are wrapped in a Node subclass that - # is compatible with the type hint. - node._wrap_value_and_set(key, subnode._value(), value_type_hint) - subnode = node._get_node(key) - assert isinstance(subnode, Node) - _deep_update_type_hint(subnode, value_type_hint) - - -def _shallow_validate_type_hint(node: Node, type_hint: Any) -> None: - """Error if node's type, content and metadata are not compatible with type_hint.""" - from omegaconf import DictConfig, ListConfig, ValueNode - - is_optional, ref_type = _resolve_optional(type_hint) - - vk = get_value_kind(node) - - if node._is_none(): - if not is_optional: - value = _get_value(node) - raise ValidationError( - f"Value {value!r} ({type(value).__name__})" - + f" is incompatible with type hint '{ref_type.__name__}'" - ) - return - elif vk in (ValueKind.MANDATORY_MISSING, ValueKind.INTERPOLATION): - return - elif vk == ValueKind.VALUE: - if is_primitive_type_annotation(ref_type) and isinstance(node, ValueNode): - value = node._value() - if not isinstance(value, ref_type): - raise ValidationError( - f"Value {value!r} ({type(value).__name__})" - + f" is incompatible with type hint '{ref_type.__name__}'" - ) - elif is_structured_config(ref_type) and isinstance(node, DictConfig): - return - elif is_dict_annotation(ref_type) and isinstance(node, DictConfig): - return - elif is_list_annotation(ref_type) and isinstance(node, ListConfig): - return - else: - if isinstance(node, ValueNode): - value = node._value() - raise ValidationError( - f"Value {value!r} ({type(value).__name__})" - + f" is incompatible with type hint '{ref_type}'" - ) - else: - raise ValidationError( - f"'{type(node).__name__}' is incompatible" + f" with type hint '{ref_type}'" - ) - - else: - assert False diff --git a/omegaconf/dictconfig.py b/omegaconf/dictconfig.py deleted file mode 100644 index 9d8310a99..000000000 --- a/omegaconf/dictconfig.py +++ /dev/null @@ -1,727 +0,0 @@ -import copy -from enum import Enum -from typing import ( - Any, - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - MutableMapping, - Optional, - Sequence, - Tuple, - Type, - Union, -) - -from ._utils import ( - _DEFAULT_MARKER_, - ValueKind, - _get_value, - _is_interpolation, - _is_missing_literal, - _is_missing_value, - _is_none, - _resolve_optional, - _valid_dict_key_annotation_type, - format_and_raise, - get_structured_config_data, - get_structured_config_init_field_names, - get_type_of, - get_value_kind, - is_container_annotation, - is_dict, - is_primitive_dict, - is_structured_config, - is_structured_config_frozen, - type_str, -) -from .base import Box, Container, ContainerMetadata, DictKeyType, Node -from .basecontainer import BaseContainer -from .errors import ( - ConfigAttributeError, - ConfigKeyError, - ConfigTypeError, - InterpolationResolutionError, - KeyValidationError, - MissingMandatoryValue, - OmegaConfBaseException, - ReadonlyConfigError, - ValidationError, -) -from .nodes import EnumNode, ValueNode - - -class DictConfig(BaseContainer, MutableMapping[Any, Any]): - _metadata: ContainerMetadata - _content: Union[Dict[DictKeyType, Node], None, str] - - def __init__( - self, - content: Union[Dict[DictKeyType, Any], "DictConfig", Any], - key: Any = None, - parent: Optional[Box] = None, - ref_type: Union[Any, Type[Any]] = Any, - key_type: Union[Any, Type[Any]] = Any, - element_type: Union[Any, Type[Any]] = Any, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ) -> None: - try: - if isinstance(content, DictConfig): - if flags is None: - flags = content._metadata.flags - super().__init__( - parent=parent, - metadata=ContainerMetadata( - key=key, - optional=is_optional, - ref_type=ref_type, - object_type=dict, - key_type=key_type, - element_type=element_type, - flags=flags, - ), - ) - - if not _valid_dict_key_annotation_type(key_type): - raise KeyValidationError(f"Unsupported key type {key_type}") - - if is_structured_config(content) or is_structured_config(ref_type): - self._set_value(content, flags=flags) - if is_structured_config_frozen(content) or is_structured_config_frozen(ref_type): - self._set_flag("readonly", True) - - else: - if isinstance(content, DictConfig): - metadata = copy.deepcopy(content._metadata) - metadata.key = key - metadata.ref_type = ref_type - metadata.optional = is_optional - metadata.element_type = element_type - metadata.key_type = key_type - self.__dict__["_metadata"] = metadata - self._set_value(content, flags=flags) - except Exception as ex: - format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex)) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "DictConfig": - res = DictConfig(None) - res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo) - res.__dict__["_flags_cache"] = copy.deepcopy(self.__dict__["_flags_cache"], memo=memo) - - src_content = self.__dict__["_content"] - if isinstance(src_content, dict): - content_copy = {} - for k, v in src_content.items(): - old_parent = v.__dict__["_parent"] - try: - v.__dict__["_parent"] = None - vc = copy.deepcopy(v, memo=memo) - vc.__dict__["_parent"] = res - content_copy[k] = vc - finally: - v.__dict__["_parent"] = old_parent - else: - # None and strings can be assigned as is - content_copy = src_content - - res.__dict__["_content"] = content_copy - # parent is retained, but not copied - res.__dict__["_parent"] = self.__dict__["_parent"] - return res - - def copy(self) -> "DictConfig": - return copy.copy(self) - - def _is_typed(self) -> bool: - return self._metadata.object_type not in (Any, None) and not is_dict(self._metadata.object_type) - - def _validate_get(self, key: Any, value: Any = None) -> None: - is_typed = self._is_typed() - - is_struct = self._get_flag("struct") is True - if key not in self.__dict__["_content"]: - if is_typed: - # do not raise an exception if struct is explicitly set to False - if self._get_node_flag("struct") is False: - return - if is_typed or is_struct: - if is_typed: - assert self._metadata.object_type not in (dict, None) - msg = f"Key '{key}' not in '{self._metadata.object_type.__name__}'" - else: - msg = f"Key '{key}' is not in struct" - self._format_and_raise(key=key, value=value, cause=ConfigAttributeError(msg)) - - def _validate_set(self, key: Any, value: Any) -> None: - from omegaconf import OmegaConf - - vk = get_value_kind(value) - if vk == ValueKind.INTERPOLATION: - return - if _is_none(value): - self._validate_non_optional(key, value) - return - if vk == ValueKind.MANDATORY_MISSING or value is None: - return - - target = self._get_node(key) if key is not None else self - - target_has_ref_type = isinstance(target, DictConfig) and target._metadata.ref_type not in (Any, dict) - is_valid_target = target is None or not target_has_ref_type - - if is_valid_target: - return - - assert isinstance(target, Node) - - target_type = target._metadata.ref_type - value_type = OmegaConf.get_type(value) - - if is_dict(value_type) and is_dict(target_type): - return - if is_container_annotation(target_type) and not is_container_annotation(value_type): - raise ValidationError(f"Cannot assign {type_str(value_type)} to {type_str(target_type)}") - - if target_type is not None and value_type is not None: - origin = getattr(target_type, "__origin__", target_type) - if not issubclass(value_type, origin): - self._raise_invalid_value(value, value_type, target_type) - - def _validate_merge(self, value: Any) -> None: - from omegaconf import OmegaConf - - dest = self - src = value - - self._validate_non_optional(None, src) - - dest_obj_type = OmegaConf.get_type(dest) - src_obj_type = OmegaConf.get_type(src) - - if dest._is_missing() and src._metadata.object_type not in (dict, None): - self._validate_set(key=None, value=_get_value(src)) - - if src._is_missing(): - return - - validation_error = ( - dest_obj_type is not None - and src_obj_type is not None - and is_structured_config(dest_obj_type) - and not src._is_none() - and not is_dict(src_obj_type) - and not issubclass(src_obj_type, dest_obj_type) - ) - if validation_error: - msg = ( - f"Merge error: {type_str(src_obj_type)} is not a " - f"subclass of {type_str(dest_obj_type)}. value: {src}" - ) - raise ValidationError(msg) - - def _validate_non_optional(self, key: Optional[DictKeyType], value: Any) -> None: - if _is_none(value, resolve=True, throw_on_resolution_failure=False): - if key is not None: - child = self._get_node(key) - if child is not None: - assert isinstance(child, Node) - field_is_optional = child._is_optional() - else: - field_is_optional, _ = _resolve_optional(self._metadata.element_type) - else: - field_is_optional = self._is_optional() - - if not field_is_optional: - self._format_and_raise( - key=key, - value=value, - cause=ValidationError("field '$FULL_KEY' is not Optional"), - ) - - def _raise_invalid_value(self, value: Any, value_type: Any, target_type: Any) -> None: - assert value_type is not None - assert target_type is not None - msg = ( - f"Invalid type assigned: {type_str(value_type)} is not a " - f"subclass of {type_str(target_type)}. value: {value}" - ) - raise ValidationError(msg) - - def _validate_and_normalize_key(self, key: Any) -> DictKeyType: - return self._s_validate_and_normalize_key(self._metadata.key_type, key) - - def _s_validate_and_normalize_key(self, key_type: Any, key: Any) -> DictKeyType: - if key_type is Any: - for t in DictKeyType.__args__: # type: ignore - if isinstance(key, t): - return key # type: ignore - raise KeyValidationError("Incompatible key type '$KEY_TYPE'") - elif key_type is bool and key in [0, 1]: - # Python treats True as 1 and False as 0 when used as dict keys - # assert hash(0) == hash(False) - # assert hash(1) == hash(True) - return bool(key) - elif key_type in (str, bytes, int, float, bool): # primitive type - if not isinstance(key, key_type): - raise KeyValidationError(f"Key $KEY ($KEY_TYPE) is incompatible with ({key_type.__name__})") - - return key # type: ignore - elif issubclass(key_type, Enum): - try: - return EnumNode.validate_and_convert_to_enum(key_type, key) - except ValidationError: - valid = ", ".join([x for x in key_type.__members__.keys()]) - raise KeyValidationError( - f"Key '$KEY' is incompatible with the enum type '{key_type.__name__}', valid: [{valid}]" - ) - else: - assert False, f"Unsupported key type {key_type}" - - def __setitem__(self, key: DictKeyType, value: Any) -> None: - try: - self.__set_impl(key=key, value=value) - except AttributeError as e: - self._format_and_raise(key=key, value=value, type_override=ConfigKeyError, cause=e) - except Exception as e: - self._format_and_raise(key=key, value=value, cause=e) - - def __set_impl(self, key: DictKeyType, value: Any) -> None: - key = self._validate_and_normalize_key(key) - self._set_item_impl(key, value) - - # hide content while inspecting in debugger - def __dir__(self) -> Iterable[str]: - if self._is_missing() or self._is_none(): - return [] - return self.__dict__["_content"].keys() # type: ignore - - def __setattr__(self, key: str, value: Any) -> None: - """ - Allow assigning attributes to DictConfig - :param key: - :param value: - :return: - """ - try: - self.__set_impl(key, value) - except Exception as e: - if isinstance(e, OmegaConfBaseException) and e._initialized: - raise e - self._format_and_raise(key=key, value=value, cause=e) - assert False - - def __getattr__(self, key: str) -> Any: - """ - Allow accessing dictionary values as attributes - :param key: - :return: - """ - if key == "__name__": - raise AttributeError() - - try: - return self._get_impl(key=key, default_value=_DEFAULT_MARKER_, validate_key=False) - except ConfigKeyError as e: - self._format_and_raise(key=key, value=None, cause=e, type_override=ConfigAttributeError) - except Exception as e: - self._format_and_raise(key=key, value=None, cause=e) - - def __getitem__(self, key: DictKeyType) -> Any: - """ - Allow map style access - :param key: - :return: - """ - - try: - return self._get_impl(key=key, default_value=_DEFAULT_MARKER_) - except AttributeError as e: - self._format_and_raise(key=key, value=None, cause=e, type_override=ConfigKeyError) - except Exception as e: - self._format_and_raise(key=key, value=None, cause=e) - - def __delattr__(self, key: str) -> None: - """ - Allow deleting dictionary values as attributes - :param key: - :return: - """ - if self._get_flag("readonly"): - self._format_and_raise( - key=key, - value=None, - cause=ReadonlyConfigError("DictConfig in read-only mode does not support deletion"), - ) - try: - del self.__dict__["_content"][key] - except KeyError: - msg = "Attribute not found: '$KEY'" - self._format_and_raise(key=key, value=None, cause=ConfigAttributeError(msg)) - - def __delitem__(self, key: DictKeyType) -> None: - key = self._validate_and_normalize_key(key) - if self._get_flag("readonly"): - self._format_and_raise( - key=key, - value=None, - cause=ReadonlyConfigError("DictConfig in read-only mode does not support deletion"), - ) - if self._get_flag("struct"): - self._format_and_raise( - key=key, - value=None, - cause=ConfigTypeError("DictConfig in struct mode does not support deletion"), - ) - if self._is_typed() and self._get_node_flag("struct") is not False: - self._format_and_raise( - key=key, - value=None, - cause=ConfigTypeError( - f"{type_str(self._metadata.object_type)} (DictConfig) does not support deletion" - ), - ) - - try: - del self.__dict__["_content"][key] - except KeyError: - msg = "Key not found: '$KEY'" - self._format_and_raise(key=key, value=None, cause=ConfigKeyError(msg)) - - def get(self, key: DictKeyType, default_value: Any = None) -> Any: - """Return the value for `key` if `key` is in the dictionary, else - `default_value` (defaulting to `None`).""" - try: - return self._get_impl(key=key, default_value=default_value) - except KeyValidationError as e: - self._format_and_raise(key=key, value=None, cause=e) - - def _get_impl(self, key: DictKeyType, default_value: Any, validate_key: bool = True) -> Any: - try: - node = self._get_child(key=key, throw_on_missing_key=True, validate_key=validate_key) - except (ConfigAttributeError, ConfigKeyError): - if default_value is not _DEFAULT_MARKER_: - return default_value - else: - raise - assert isinstance(node, Node) - return self._resolve_with_default(key=key, value=node, default_value=default_value) - - def _get_node( - self, - key: DictKeyType, - validate_access: bool = True, - validate_key: bool = True, - throw_on_missing_value: bool = False, - throw_on_missing_key: bool = False, - ) -> Optional[Node]: - try: - key = self._validate_and_normalize_key(key) - except KeyValidationError: - if validate_access and validate_key: - raise - else: - if throw_on_missing_key: - raise ConfigAttributeError - else: - return None - - if validate_access: - self._validate_get(key) - - value: Optional[Node] = self.__dict__["_content"].get(key) - if value is None: - if throw_on_missing_key: - raise ConfigKeyError(f"Missing key {key!s}") - elif throw_on_missing_value and value._is_missing(): - raise MissingMandatoryValue("Missing mandatory value: $KEY") - return value - - def pop(self, key: DictKeyType, default: Any = _DEFAULT_MARKER_) -> Any: - try: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot pop from read-only node") - if self._get_flag("struct"): - raise ConfigTypeError("DictConfig in struct mode does not support pop") - if self._is_typed() and self._get_node_flag("struct") is not False: - raise ConfigTypeError( - f"{type_str(self._metadata.object_type)} (DictConfig) does not support pop" - ) - key = self._validate_and_normalize_key(key) - node = self._get_child(key=key, validate_access=False) - if node is not None: - assert isinstance(node, Node) - value = self._resolve_with_default(key=key, value=node, default_value=default) - - del self[key] - return value - else: - if default is not _DEFAULT_MARKER_: - return default - else: - full = self._get_full_key(key=key) - if full != key: - raise ConfigKeyError(f"Key not found: '{key!s}' (path: '{full}')") - else: - raise ConfigKeyError(f"Key not found: '{key!s}'") - except Exception as e: - self._format_and_raise(key=key, value=None, cause=e) - - def keys(self) -> KeysView[DictKeyType]: - if self._is_missing() or self._is_interpolation() or self._is_none(): - return {}.keys() - ret = self.__dict__["_content"].keys() - assert isinstance(ret, KeysView) - return ret - - def __contains__(self, key: object) -> bool: - """ - A key is contained in a DictConfig if there is an associated value and - it is not a mandatory missing value ('???'). - :param key: - :return: - """ - - try: - key = self._validate_and_normalize_key(key) - except KeyValidationError: - return False - - try: - node = self._get_child(key) - assert node is None or isinstance(node, Node) - except (KeyError, AttributeError): - node = None - - if node is None: - return False - else: - try: - self._resolve_with_default(key=key, value=node) - return True - except InterpolationResolutionError: - # Interpolations that fail count as existing. - return True - except MissingMandatoryValue: - # Missing values count as *not* existing. - return False - - def __iter__(self) -> Iterator[DictKeyType]: - return iter(self.keys()) - - def items(self) -> ItemsView[DictKeyType, Any]: - return dict(self.items_ex(resolve=True, keys=None)).items() - - def setdefault(self, key: DictKeyType, default: Any = None) -> Any: - if key in self: - ret = self.__getitem__(key) - else: - ret = default - self.__setitem__(key, default) - return ret - - def items_ex( - self, resolve: bool = True, keys: Optional[Sequence[DictKeyType]] = None - ) -> List[Tuple[DictKeyType, Any]]: - items: List[Tuple[DictKeyType, Any]] = [] - - if self._is_none(): - self._format_and_raise( - key=None, - value=None, - cause=TypeError("Cannot iterate a DictConfig object representing None"), - ) - if self._is_missing(): - raise MissingMandatoryValue("Cannot iterate a missing DictConfig") - - for key in self.keys(): - if resolve: - value = self[key] - else: - value = self.__dict__["_content"][key] - if isinstance(value, ValueNode): - value = value._value() - if keys is None or key in keys: - items.append((key, value)) - - return items - - def __eq__(self, other: Any) -> bool: - if other is None: - return self.__dict__["_content"] is None - if is_primitive_dict(other) or is_structured_config(other): - other = DictConfig(other, flags={"allow_objects": True}) - return DictConfig._dict_conf_eq(self, other) - if isinstance(other, DictConfig): - return DictConfig._dict_conf_eq(self, other) - if self._is_missing(): - return _is_missing_literal(other) - return NotImplemented - - def __ne__(self, other: Any) -> bool: - x = self.__eq__(other) - if x is not NotImplemented: - return not x - return NotImplemented - - def __hash__(self) -> int: - return hash(str(self)) - - def _promote(self, type_or_prototype: Optional[Type[Any]]) -> None: - """ - Retypes a node. - This should only be used in rare circumstances, where you want to dynamically change - the runtime structured-type of a DictConfig. - It will change the type and add the additional fields based on the input class or object - """ - if type_or_prototype is None: - return - if not is_structured_config(type_or_prototype): - raise ValueError(f"Expected structured config class: {type_or_prototype}") - - from omegaconf import OmegaConf - - proto: DictConfig = OmegaConf.structured(type_or_prototype) - object_type = proto._metadata.object_type - # remove the type to prevent assignment validation from rejecting the promotion. - proto._metadata.object_type = None - self.merge_with(proto) - # restore the type. - self._metadata.object_type = object_type - - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - try: - previous_content = self.__dict__["_content"] - self._set_value_impl(value, flags) - except Exception as e: - self.__dict__["_content"] = previous_content - raise e - - def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - from omegaconf import MISSING, flag_override - - if flags is None: - flags = {} - - assert not isinstance(value, ValueNode) - self._validate_set(key=None, value=value) - - if _is_none(value, resolve=True): - self.__dict__["_content"] = None - self._metadata.object_type = None - elif _is_interpolation(value, strict_interpolation_validation=True): - self.__dict__["_content"] = value - self._metadata.object_type = None - elif _is_missing_value(value): - self.__dict__["_content"] = MISSING - self._metadata.object_type = None - else: - self.__dict__["_content"] = {} - if is_structured_config(value): - self._metadata.object_type = None - ao = self._get_flag("allow_objects") - data = get_structured_config_data(value, allow_objects=ao) - with flag_override(self, ["struct", "readonly"], False): - for k, v in data.items(): - self.__setitem__(k, v) - self._metadata.object_type = get_type_of(value) - - elif isinstance(value, DictConfig): - self._metadata.flags = copy.deepcopy(flags) - with flag_override(self, ["struct", "readonly"], False): - for k, v in value.__dict__["_content"].items(): - self.__setitem__(k, v) - self._metadata.object_type = value._metadata.object_type - - elif isinstance(value, dict): - with flag_override(self, ["struct", "readonly"], False): - for k, v in value.items(): - self.__setitem__(k, v) - self._metadata.object_type = dict - - else: # pragma: no cover - msg = f"Unsupported value type: {value}" - raise ValidationError(msg) - - @staticmethod - def _dict_conf_eq(d1: "DictConfig", d2: "DictConfig") -> bool: - d1_none = d1.__dict__["_content"] is None - d2_none = d2.__dict__["_content"] is None - if d1_none and d2_none: - return True - if d1_none != d2_none: - return False - - assert isinstance(d1, DictConfig) - assert isinstance(d2, DictConfig) - if len(d1) != len(d2): - return False - if d1._is_missing() or d2._is_missing(): - return d1._is_missing() is d2._is_missing() - - for k, v in d1.items_ex(resolve=False): - if k not in d2.__dict__["_content"]: - return False - if not BaseContainer._item_eq(d1, k, d2, k): - return False - - return True - - def _to_object(self) -> Any: - """ - Instantiate an instance of `self._metadata.object_type`. - This requires `self` to be a structured config. - Nested subconfigs are converted by calling `OmegaConf.to_object`. - """ - from omegaconf import OmegaConf - - object_type = self._metadata.object_type - assert is_structured_config(object_type) - init_field_names = set(get_structured_config_init_field_names(object_type)) - - init_field_items: Dict[str, Any] = {} - non_init_field_items: Dict[str, Any] = {} - for k in self.keys(): - assert isinstance(k, str) - node = self._get_child(k) - assert isinstance(node, Node) - try: - node = node._dereference_node() - except InterpolationResolutionError as e: - self._format_and_raise(key=k, value=None, cause=e) - if node._is_missing(): - if k not in init_field_names: - continue # MISSING is ignored for init=False fields - self._format_and_raise( - key=k, - value=None, - cause=MissingMandatoryValue( - "Structured config of type `$OBJECT_TYPE` has missing mandatory value: $KEY" - ), - ) - if isinstance(node, Container): - v = OmegaConf.to_object(node) - else: - v = node._value() - - if k in init_field_names: - init_field_items[k] = v - else: - non_init_field_items[k] = v - - try: - result = object_type(**init_field_items) - except TypeError as exc: - self._format_and_raise( - key=None, - value=None, - cause=exc, - msg="Could not create instance of `$OBJECT_TYPE`: " + str(exc), - ) - - for k, v in non_init_field_items.items(): - setattr(result, k, v) - return result diff --git a/omegaconf/errors.py b/omegaconf/errors.py deleted file mode 100644 index 60525ef3c..000000000 --- a/omegaconf/errors.py +++ /dev/null @@ -1,141 +0,0 @@ -from typing import Any, Optional, Type - - -class OmegaConfBaseException(Exception): - # would ideally be typed Optional[Node] - parent_node: Any - child_node: Any - key: Any - full_key: Optional[str] - value: Any - msg: Optional[str] - cause: Optional[Exception] - object_type: Optional[Type[Any]] - object_type_str: Optional[str] - ref_type: Optional[Type[Any]] - ref_type_str: Optional[str] - - _initialized: bool = False - - def __init__(self, *_args: Any, **_kwargs: Any) -> None: - self.parent_node = None - self.child_node = None - self.key = None - self.full_key = None - self.value = None - self.msg = None - self.object_type = None - self.ref_type = None - - -class MissingMandatoryValue(OmegaConfBaseException): - """Thrown when a variable flagged with '???' value is accessed to - indicate that the value was not set""" - - -class KeyValidationError(OmegaConfBaseException, ValueError): - """ - Thrown when an a key of invalid type is used - """ - - -class ValidationError(OmegaConfBaseException, ValueError): - """ - Thrown when a value fails validation - """ - - -class UnsupportedValueType(ValidationError, ValueError): - """ - Thrown when an input value is not of supported type - """ - - -class ReadonlyConfigError(OmegaConfBaseException): - """ - Thrown when someone tries to modify a frozen config - """ - - -class InterpolationResolutionError(OmegaConfBaseException, ValueError): - """ - Base class for exceptions raised when resolving an interpolation. - """ - - -class UnsupportedInterpolationType(InterpolationResolutionError): - """ - Thrown when an attempt to use an unregistered interpolation is made - """ - - -class InterpolationKeyError(InterpolationResolutionError): - """ - Thrown when a node does not exist when resolving an interpolation. - """ - - -class InterpolationToMissingValueError(InterpolationResolutionError): - """ - Thrown when a node interpolation points to a node that is set to ???. - """ - - -class InterpolationValidationError(InterpolationResolutionError, ValidationError): - """ - Thrown when the result of an interpolation fails the validation step. - """ - - -class ConfigKeyError(OmegaConfBaseException, KeyError): - """ - Thrown from DictConfig when a regular dict access would have caused a KeyError. - """ - - msg: str - - def __init__(self, msg: str) -> None: - super().__init__(msg) - self.msg = msg - - def __str__(self) -> str: - """ - Workaround to nasty KeyError quirk: https://bugs.python.org/issue2651 - """ - return self.msg - - -class ConfigAttributeError(OmegaConfBaseException, AttributeError): - """ - Thrown from a config object when a regular access would have caused an AttributeError. - """ - - -class ConfigTypeError(OmegaConfBaseException, TypeError): - """ - Thrown from a config object when a regular access would have caused a TypeError. - """ - - -class ConfigIndexError(OmegaConfBaseException, IndexError): - """ - Thrown from a config object when a regular access would have caused an IndexError. - """ - - -class ConfigValueError(OmegaConfBaseException, ValueError): - """ - Thrown from a config object when a regular access would have caused a ValueError. - """ - - -class ConfigCycleDetectedException(OmegaConfBaseException): - """ - Thrown when a cycle is detected in the graph made by config nodes. - """ - - -class GrammarParseError(OmegaConfBaseException): - """ - Thrown when failing to parse an expression according to the ANTLR grammar. - """ diff --git a/omegaconf/grammar/OmegaConfGrammarLexer.g4 b/omegaconf/grammar/OmegaConfGrammarLexer.g4 deleted file mode 100644 index f7ab2044a..000000000 --- a/omegaconf/grammar/OmegaConfGrammarLexer.g4 +++ /dev/null @@ -1,137 +0,0 @@ -// Regenerate lexer and parser by running 'python setup.py antlr' at project root. -// See `OmegaConfGrammarParser.g4` for some important information regarding how to -// properly maintain this grammar. - -lexer grammar OmegaConfGrammarLexer; - -// Re-usable fragments. -fragment CHAR: [a-zA-Z]; -fragment DIGIT: [0-9]; -fragment INT_UNSIGNED: '0' | [1-9] (('_')? DIGIT)*; -fragment ESC_BACKSLASH: '\\\\'; // escaped backslash - -///////////////////////////// -// DEFAULT_MODE (TOPLEVEL) // -///////////////////////////// - -TOP_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); - -// Regular string: anything that does not contain any $ and does not end with \ -// (this ensures this rule will not consume characters required to recognize other tokens). -ANY_STR: ~[$]* ~[\\$]; - -// Escaped interpolation: '\${', optionally preceded by an even number of \ -ESC_INTER: ESC_BACKSLASH* '\\${'; - -// Backslashes that *may* be escaped (even number). -TOP_ESC: ESC_BACKSLASH+; - -// Other backslashes that will not need escaping (odd number due to not matching the previous rule). -BACKSLASHES: '\\'+ -> type(ANY_STR); - -// The dollar sign must be singled out so that we can recognize interpolations. -DOLLAR: '$' -> type(ANY_STR); - - -//////////////// -// VALUE_MODE // -//////////////// - -mode VALUE_MODE; - -INTER_OPEN: '${' WS? -> pushMode(INTERPOLATION_MODE); -BRACE_OPEN: '{' WS? -> pushMode(VALUE_MODE); // must keep track of braces to detect end of interpolation -BRACE_CLOSE: WS? '}' -> popMode; -QUOTE_OPEN_SINGLE: '\'' -> pushMode(QUOTED_SINGLE_MODE); -QUOTE_OPEN_DOUBLE: '"' -> pushMode(QUOTED_DOUBLE_MODE); - -COMMA: WS? ',' WS?; -BRACKET_OPEN: '[' WS?; -BRACKET_CLOSE: WS? ']'; -COLON: WS? ':' WS?; - -// Numbers. - -fragment POINT_FLOAT: INT_UNSIGNED '.' | INT_UNSIGNED? '.' DIGIT (('_')? DIGIT)*; -fragment EXPONENT_FLOAT: (INT_UNSIGNED | POINT_FLOAT) [eE] [+-]? DIGIT (('_')? DIGIT)*; -FLOAT: [+-]? (POINT_FLOAT | EXPONENT_FLOAT | [Ii][Nn][Ff] | [Nn][Aa][Nn]); -INT: [+-]? INT_UNSIGNED; - -// Other reserved keywords. - -BOOL: - [Tt][Rr][Uu][Ee] // TRUE - | [Ff][Aa][Ll][Ss][Ee]; // FALSE - -NULL: [Nn][Uu][Ll][Ll]; - -UNQUOTED_CHAR: [/\-\\+.$%*@?|]; // other characters allowed in unquoted strings -ID: (CHAR|'_') (CHAR|DIGIT|'_'|'-')*; -ESC: (ESC_BACKSLASH | '\\(' | '\\)' | '\\[' | '\\]' | '\\{' | '\\}' | - '\\:' | '\\=' | '\\,' | '\\ ' | '\\\t')+; -WS: [ \t]+; - - -//////////////////////// -// INTERPOLATION_MODE // -//////////////////////// - -mode INTERPOLATION_MODE; - -NESTED_INTER_OPEN: INTER_OPEN WS? -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); -INTER_COLON: WS? ':' WS? -> type(COLON), mode(VALUE_MODE); -INTER_CLOSE: WS? '}' -> popMode; - -DOT: '.'; -INTER_BRACKET_OPEN: '[' -> type(BRACKET_OPEN); -INTER_BRACKET_CLOSE: ']' -> type(BRACKET_CLOSE); -INTER_ID: ID -> type(ID); - -// Interpolation key, may contain any non special character. -// Note that we can allow '$' because the parser does not support interpolations that -// are only part of a key name, i.e., "${foo${bar}}" is not allowed. As a result, it -// is ok to "consume" all '$' characters within the `INTER_KEY` token. -INTER_KEY: ~[\\{}()[\]:. \t'"]+; - - -//////////////////////// -// QUOTED_SINGLE_MODE // -//////////////////////// - -mode QUOTED_SINGLE_MODE; - -// This mode is very similar to `DEFAULT_MODE` except for the handling of quotes. - -QSINGLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); -MATCHING_QUOTE_CLOSE: '\'' -> popMode; - -// Regular string: anything that does not contain any $ *or quote* and does not end with \ -QSINGLE_STR: ~['$]* ~['\\$] -> type(ANY_STR); - -QSINGLE_ESC_INTER: ESC_INTER -> type(ESC_INTER); - -// Escaped quote (optionally preceded by an even number of backslashes). -QSINGLE_ESC_QUOTE: ESC_BACKSLASH* '\\\'' -> type(ESC); - -QUOTED_ESC: ESC_BACKSLASH+; -QSINGLE_BACKSLASHES: '\\'+ -> type(ANY_STR); -QSINGLE_DOLLAR: '$' -> type(ANY_STR); - - -//////////////////////// -// QUOTED_DOUBLE_MODE // -//////////////////////// - -mode QUOTED_DOUBLE_MODE; - -// Same as `QUOTED_SINGLE_MODE` but for double quotes. - -QDOUBLE_INTER_OPEN: INTER_OPEN -> type(INTER_OPEN), pushMode(INTERPOLATION_MODE); -QDOUBLE_CLOSE: '"' -> type(MATCHING_QUOTE_CLOSE), popMode; - -QDOUBLE_STR: ~["$]* ~["\\$] -> type(ANY_STR); -QDOUBLE_ESC_INTER: ESC_INTER -> type(ESC_INTER); -QDOUBLE_ESC_QUOTE: ESC_BACKSLASH* '\\"' -> type(ESC); -QDOUBLE_ESC: ESC_BACKSLASH+ -> type(QUOTED_ESC); -QDOUBLE_BACKSLASHES: '\\'+ -> type(ANY_STR); -QDOUBLE_DOLLAR: '$' -> type(ANY_STR); diff --git a/omegaconf/grammar/OmegaConfGrammarParser.g4 b/omegaconf/grammar/OmegaConfGrammarParser.g4 deleted file mode 100644 index 040692c8d..000000000 --- a/omegaconf/grammar/OmegaConfGrammarParser.g4 +++ /dev/null @@ -1,91 +0,0 @@ -// Regenerate parser by running 'python setup.py antlr' at project root. - -// Maintenance guidelines when modifying this grammar: -// -// - Consider whether the regex pattern `SIMPLE_INTERPOLATION_PATTERN` found in -// `grammar_parser.py` should be updated as well. -// -// - Update Hydra's grammar accordingly. -// -// - Keep up-to-date the comments in the visitor (in `grammar_visitor.py`) -// that contain grammar excerpts (within each `visit...()` method). -// -// - Remember to update the documentation (including the tutorial notebook as -// well as grammar.rst) - -parser grammar OmegaConfGrammarParser; -options {tokenVocab = OmegaConfGrammarLexer;} - -// Main rules used to parse OmegaConf strings. - -configValue: text EOF; -singleElement: element EOF; - - -// Composite text expression (may contain interpolations). - -text: (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+; - - -// Elements. - -element: - primitive - | quotedValue - | listContainer - | dictContainer -; - - -// Data structures. - -listContainer: BRACKET_OPEN sequence? BRACKET_CLOSE; // [], [1,2,3], [a,b,[1,2]] -dictContainer: BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE; // {}, {a:10,b:20} -dictKeyValuePair: dictKey COLON element; -sequence: (element (COMMA element?)*) | (COMMA element?)+; - - -// Interpolations. - -interpolation: interpolationNode | interpolationResolver; - -interpolationNode: - INTER_OPEN - DOT* // relative interpolation? - (configKey | BRACKET_OPEN configKey BRACKET_CLOSE) // foo, [foo] - (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)* // .foo, [foo], .foo[bar], [foo].bar[baz] - INTER_CLOSE; -interpolationResolver: INTER_OPEN resolverName COLON sequence? BRACE_CLOSE; -configKey: interpolation | ID | INTER_KEY; -resolverName: (interpolation | ID) (DOT (interpolation | ID))* ; // oc.env, myfunc, ns.${x}, ns1.ns2.f - - -// Primitive types. - -// Ex: "hello world", 'hello ${world}' -quotedValue: (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE; - -primitive: - ( ID // foo_10 - | NULL // null, NULL - | INT // 0, 10, -20, 1_000_000 - | FLOAT // 3.14, -20.0, 1e-1, -10e3 - | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | - | COLON // : - | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, - | WS // whitespaces - | interpolation - )+; - -// Same as `primitive` except that `COLON` and interpolations are not allowed. -dictKey: - ( ID // foo_10 - | NULL // null, NULL - | INT // 0, 10, -20, 1_000_000 - | FLOAT // 3.14, -20.0, 1e-1, -10e3 - | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | - | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, - | WS // whitespaces - )+; \ No newline at end of file diff --git a/omegaconf/grammar/__init__.py b/omegaconf/grammar/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp b/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp deleted file mode 100644 index 98152df6e..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarLexer.interp +++ /dev/null @@ -1,130 +0,0 @@ -token literal names: -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -'.' -null -null -null -null -'[' -']' - -token symbolic names: -null -ANY_STR -ESC_INTER -TOP_ESC -INTER_OPEN -BRACE_OPEN -BRACE_CLOSE -QUOTE_OPEN_SINGLE -QUOTE_OPEN_DOUBLE -COMMA -BRACKET_OPEN -BRACKET_CLOSE -COLON -FLOAT -INT -BOOL -NULL -UNQUOTED_CHAR -ID -ESC -WS -INTER_CLOSE -DOT -INTER_KEY -MATCHING_QUOTE_CLOSE -QUOTED_ESC -DOLLAR -INTER_BRACKET_OPEN -INTER_BRACKET_CLOSE - -rule names: -CHAR -DIGIT -INT_UNSIGNED -ESC_BACKSLASH -TOP_INTER_OPEN -ANY_STR -ESC_INTER -TOP_ESC -BACKSLASHES -DOLLAR -INTER_OPEN -BRACE_OPEN -BRACE_CLOSE -QUOTE_OPEN_SINGLE -QUOTE_OPEN_DOUBLE -COMMA -BRACKET_OPEN -BRACKET_CLOSE -COLON -POINT_FLOAT -EXPONENT_FLOAT -FLOAT -INT -BOOL -NULL -UNQUOTED_CHAR -ID -ESC -WS -NESTED_INTER_OPEN -INTER_COLON -INTER_CLOSE -DOT -INTER_BRACKET_OPEN -INTER_BRACKET_CLOSE -INTER_ID -INTER_KEY -QSINGLE_INTER_OPEN -MATCHING_QUOTE_CLOSE -QSINGLE_STR -QSINGLE_ESC_INTER -QSINGLE_ESC_QUOTE -QUOTED_ESC -QSINGLE_BACKSLASHES -QSINGLE_DOLLAR -QDOUBLE_INTER_OPEN -QDOUBLE_CLOSE -QDOUBLE_STR -QDOUBLE_ESC_INTER -QDOUBLE_ESC_QUOTE -QDOUBLE_ESC -QDOUBLE_BACKSLASHES -QDOUBLE_DOLLAR - -channel names: -DEFAULT_TOKEN_CHANNEL -HIDDEN - -mode names: -DEFAULT_MODE -VALUE_MODE -INTERPOLATION_MODE -QUOTED_SINGLE_MODE -QUOTED_DOUBLE_MODE - -atn: -[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 30, 487, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 4, 35, 9, 35, 4, 36, 9, 36, 4, 37, 9, 37, 4, 38, 9, 38, 4, 39, 9, 39, 4, 40, 9, 40, 4, 41, 9, 41, 4, 42, 9, 42, 4, 43, 9, 43, 4, 44, 9, 44, 4, 45, 9, 45, 4, 46, 9, 46, 4, 47, 9, 47, 4, 48, 9, 48, 4, 49, 9, 49, 4, 50, 9, 50, 4, 51, 9, 51, 4, 52, 9, 52, 4, 53, 9, 53, 4, 54, 9, 54, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 5, 4, 121, 10, 4, 3, 4, 7, 4, 124, 10, 4, 12, 4, 14, 4, 127, 11, 4, 5, 4, 129, 10, 4, 3, 5, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 7, 7, 7, 140, 10, 7, 12, 7, 14, 7, 143, 11, 7, 3, 7, 3, 7, 3, 8, 7, 8, 148, 10, 8, 12, 8, 14, 8, 151, 11, 8, 3, 8, 3, 8, 3, 8, 3, 8, 3, 9, 6, 9, 158, 10, 9, 13, 9, 14, 9, 159, 3, 10, 6, 10, 163, 10, 10, 13, 10, 14, 10, 164, 3, 10, 3, 10, 3, 11, 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 5, 12, 177, 10, 12, 3, 12, 3, 12, 3, 13, 3, 13, 5, 13, 183, 10, 13, 3, 13, 3, 13, 3, 14, 5, 14, 188, 10, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 15, 3, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 17, 5, 17, 203, 10, 17, 3, 17, 3, 17, 5, 17, 207, 10, 17, 3, 18, 3, 18, 5, 18, 211, 10, 18, 3, 19, 5, 19, 214, 10, 19, 3, 19, 3, 19, 3, 20, 5, 20, 219, 10, 20, 3, 20, 3, 20, 5, 20, 223, 10, 20, 3, 21, 3, 21, 3, 21, 3, 21, 5, 21, 229, 10, 21, 3, 21, 3, 21, 3, 21, 5, 21, 234, 10, 21, 3, 21, 7, 21, 237, 10, 21, 12, 21, 14, 21, 240, 11, 21, 5, 21, 242, 10, 21, 3, 22, 3, 22, 5, 22, 246, 10, 22, 3, 22, 3, 22, 5, 22, 250, 10, 22, 3, 22, 3, 22, 5, 22, 254, 10, 22, 3, 22, 7, 22, 257, 10, 22, 12, 22, 14, 22, 260, 11, 22, 3, 23, 5, 23, 263, 10, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 5, 23, 273, 10, 23, 3, 24, 5, 24, 276, 10, 24, 3, 24, 3, 24, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 5, 25, 289, 10, 25, 3, 26, 3, 26, 3, 26, 3, 26, 3, 26, 3, 27, 3, 27, 3, 28, 3, 28, 5, 28, 300, 10, 28, 3, 28, 3, 28, 3, 28, 7, 28, 305, 10, 28, 12, 28, 14, 28, 308, 11, 28, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 3, 29, 6, 29, 333, 10, 29, 13, 29, 14, 29, 334, 3, 30, 6, 30, 338, 10, 30, 13, 30, 14, 30, 339, 3, 31, 3, 31, 5, 31, 344, 10, 31, 3, 31, 3, 31, 3, 31, 3, 32, 5, 32, 350, 10, 32, 3, 32, 3, 32, 5, 32, 354, 10, 32, 3, 32, 3, 32, 3, 32, 3, 33, 5, 33, 360, 10, 33, 3, 33, 3, 33, 3, 33, 3, 33, 3, 34, 3, 34, 3, 35, 3, 35, 3, 35, 3, 35, 3, 36, 3, 36, 3, 36, 3, 36, 3, 37, 3, 37, 3, 37, 3, 37, 3, 38, 6, 38, 381, 10, 38, 13, 38, 14, 38, 382, 3, 39, 3, 39, 3, 39, 3, 39, 3, 39, 3, 40, 3, 40, 3, 40, 3, 40, 3, 41, 7, 41, 395, 10, 41, 12, 41, 14, 41, 398, 11, 41, 3, 41, 3, 41, 3, 41, 3, 41, 3, 42, 3, 42, 3, 42, 3, 42, 3, 43, 7, 43, 409, 10, 43, 12, 43, 14, 43, 412, 11, 43, 3, 43, 3, 43, 3, 43, 3, 43, 3, 43, 3, 44, 6, 44, 420, 10, 44, 13, 44, 14, 44, 421, 3, 45, 6, 45, 425, 10, 45, 13, 45, 14, 45, 426, 3, 45, 3, 45, 3, 46, 3, 46, 3, 46, 3, 46, 3, 47, 3, 47, 3, 47, 3, 47, 3, 47, 3, 48, 3, 48, 3, 48, 3, 48, 3, 48, 3, 49, 7, 49, 446, 10, 49, 12, 49, 14, 49, 449, 11, 49, 3, 49, 3, 49, 3, 49, 3, 49, 3, 50, 3, 50, 3, 50, 3, 50, 3, 51, 7, 51, 460, 10, 51, 12, 51, 14, 51, 463, 11, 51, 3, 51, 3, 51, 3, 51, 3, 51, 3, 51, 3, 52, 6, 52, 471, 10, 52, 13, 52, 14, 52, 472, 3, 52, 3, 52, 3, 53, 6, 53, 478, 10, 53, 13, 53, 14, 53, 479, 3, 53, 3, 53, 3, 54, 3, 54, 3, 54, 3, 54, 2, 2, 55, 7, 2, 9, 2, 11, 2, 13, 2, 15, 2, 17, 3, 19, 4, 21, 5, 23, 2, 25, 28, 27, 6, 29, 7, 31, 8, 33, 9, 35, 10, 37, 11, 39, 12, 41, 13, 43, 14, 45, 2, 47, 2, 49, 15, 51, 16, 53, 17, 55, 18, 57, 19, 59, 20, 61, 21, 63, 22, 65, 2, 67, 2, 69, 23, 71, 24, 73, 29, 75, 30, 77, 2, 79, 25, 81, 2, 83, 26, 85, 2, 87, 2, 89, 2, 91, 27, 93, 2, 95, 2, 97, 2, 99, 2, 101, 2, 103, 2, 105, 2, 107, 2, 109, 2, 111, 2, 7, 2, 3, 4, 5, 6, 26, 4, 2, 67, 92, 99, 124, 3, 2, 50, 59, 3, 2, 51, 59, 3, 2, 38, 38, 4, 2, 38, 38, 94, 94, 4, 2, 71, 71, 103, 103, 4, 2, 45, 45, 47, 47, 4, 2, 75, 75, 107, 107, 4, 2, 80, 80, 112, 112, 4, 2, 72, 72, 104, 104, 4, 2, 67, 67, 99, 99, 4, 2, 86, 86, 118, 118, 4, 2, 84, 84, 116, 116, 4, 2, 87, 87, 119, 119, 4, 2, 78, 78, 110, 110, 4, 2, 85, 85, 117, 117, 8, 2, 38, 39, 44, 45, 47, 49, 65, 66, 94, 94, 126, 126, 4, 2, 47, 47, 97, 97, 4, 2, 11, 11, 34, 34, 11, 2, 11, 11, 34, 34, 36, 36, 41, 43, 48, 48, 60, 60, 93, 95, 125, 125, 127, 127, 4, 2, 38, 38, 41, 41, 5, 2, 38, 38, 41, 41, 94, 94, 4, 2, 36, 36, 38, 38, 5, 2, 36, 36, 38, 38, 94, 94, 2, 536, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 3, 27, 3, 2, 2, 2, 3, 29, 3, 2, 2, 2, 3, 31, 3, 2, 2, 2, 3, 33, 3, 2, 2, 2, 3, 35, 3, 2, 2, 2, 3, 37, 3, 2, 2, 2, 3, 39, 3, 2, 2, 2, 3, 41, 3, 2, 2, 2, 3, 43, 3, 2, 2, 2, 3, 49, 3, 2, 2, 2, 3, 51, 3, 2, 2, 2, 3, 53, 3, 2, 2, 2, 3, 55, 3, 2, 2, 2, 3, 57, 3, 2, 2, 2, 3, 59, 3, 2, 2, 2, 3, 61, 3, 2, 2, 2, 3, 63, 3, 2, 2, 2, 4, 65, 3, 2, 2, 2, 4, 67, 3, 2, 2, 2, 4, 69, 3, 2, 2, 2, 4, 71, 3, 2, 2, 2, 4, 73, 3, 2, 2, 2, 4, 75, 3, 2, 2, 2, 4, 77, 3, 2, 2, 2, 4, 79, 3, 2, 2, 2, 5, 81, 3, 2, 2, 2, 5, 83, 3, 2, 2, 2, 5, 85, 3, 2, 2, 2, 5, 87, 3, 2, 2, 2, 5, 89, 3, 2, 2, 2, 5, 91, 3, 2, 2, 2, 5, 93, 3, 2, 2, 2, 5, 95, 3, 2, 2, 2, 6, 97, 3, 2, 2, 2, 6, 99, 3, 2, 2, 2, 6, 101, 3, 2, 2, 2, 6, 103, 3, 2, 2, 2, 6, 105, 3, 2, 2, 2, 6, 107, 3, 2, 2, 2, 6, 109, 3, 2, 2, 2, 6, 111, 3, 2, 2, 2, 7, 113, 3, 2, 2, 2, 9, 115, 3, 2, 2, 2, 11, 128, 3, 2, 2, 2, 13, 130, 3, 2, 2, 2, 15, 133, 3, 2, 2, 2, 17, 141, 3, 2, 2, 2, 19, 149, 3, 2, 2, 2, 21, 157, 3, 2, 2, 2, 23, 162, 3, 2, 2, 2, 25, 168, 3, 2, 2, 2, 27, 172, 3, 2, 2, 2, 29, 180, 3, 2, 2, 2, 31, 187, 3, 2, 2, 2, 33, 193, 3, 2, 2, 2, 35, 197, 3, 2, 2, 2, 37, 202, 3, 2, 2, 2, 39, 208, 3, 2, 2, 2, 41, 213, 3, 2, 2, 2, 43, 218, 3, 2, 2, 2, 45, 241, 3, 2, 2, 2, 47, 245, 3, 2, 2, 2, 49, 262, 3, 2, 2, 2, 51, 275, 3, 2, 2, 2, 53, 288, 3, 2, 2, 2, 55, 290, 3, 2, 2, 2, 57, 295, 3, 2, 2, 2, 59, 299, 3, 2, 2, 2, 61, 332, 3, 2, 2, 2, 63, 337, 3, 2, 2, 2, 65, 341, 3, 2, 2, 2, 67, 349, 3, 2, 2, 2, 69, 359, 3, 2, 2, 2, 71, 365, 3, 2, 2, 2, 73, 367, 3, 2, 2, 2, 75, 371, 3, 2, 2, 2, 77, 375, 3, 2, 2, 2, 79, 380, 3, 2, 2, 2, 81, 384, 3, 2, 2, 2, 83, 389, 3, 2, 2, 2, 85, 396, 3, 2, 2, 2, 87, 403, 3, 2, 2, 2, 89, 410, 3, 2, 2, 2, 91, 419, 3, 2, 2, 2, 93, 424, 3, 2, 2, 2, 95, 430, 3, 2, 2, 2, 97, 434, 3, 2, 2, 2, 99, 439, 3, 2, 2, 2, 101, 447, 3, 2, 2, 2, 103, 454, 3, 2, 2, 2, 105, 461, 3, 2, 2, 2, 107, 470, 3, 2, 2, 2, 109, 477, 3, 2, 2, 2, 111, 483, 3, 2, 2, 2, 113, 114, 9, 2, 2, 2, 114, 8, 3, 2, 2, 2, 115, 116, 9, 3, 2, 2, 116, 10, 3, 2, 2, 2, 117, 129, 7, 50, 2, 2, 118, 125, 9, 4, 2, 2, 119, 121, 7, 97, 2, 2, 120, 119, 3, 2, 2, 2, 120, 121, 3, 2, 2, 2, 121, 122, 3, 2, 2, 2, 122, 124, 5, 9, 3, 2, 123, 120, 3, 2, 2, 2, 124, 127, 3, 2, 2, 2, 125, 123, 3, 2, 2, 2, 125, 126, 3, 2, 2, 2, 126, 129, 3, 2, 2, 2, 127, 125, 3, 2, 2, 2, 128, 117, 3, 2, 2, 2, 128, 118, 3, 2, 2, 2, 129, 12, 3, 2, 2, 2, 130, 131, 7, 94, 2, 2, 131, 132, 7, 94, 2, 2, 132, 14, 3, 2, 2, 2, 133, 134, 5, 27, 12, 2, 134, 135, 3, 2, 2, 2, 135, 136, 8, 6, 2, 2, 136, 137, 8, 6, 3, 2, 137, 16, 3, 2, 2, 2, 138, 140, 10, 5, 2, 2, 139, 138, 3, 2, 2, 2, 140, 143, 3, 2, 2, 2, 141, 139, 3, 2, 2, 2, 141, 142, 3, 2, 2, 2, 142, 144, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 144, 145, 10, 6, 2, 2, 145, 18, 3, 2, 2, 2, 146, 148, 5, 13, 5, 2, 147, 146, 3, 2, 2, 2, 148, 151, 3, 2, 2, 2, 149, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 152, 3, 2, 2, 2, 151, 149, 3, 2, 2, 2, 152, 153, 7, 94, 2, 2, 153, 154, 7, 38, 2, 2, 154, 155, 7, 125, 2, 2, 155, 20, 3, 2, 2, 2, 156, 158, 5, 13, 5, 2, 157, 156, 3, 2, 2, 2, 158, 159, 3, 2, 2, 2, 159, 157, 3, 2, 2, 2, 159, 160, 3, 2, 2, 2, 160, 22, 3, 2, 2, 2, 161, 163, 7, 94, 2, 2, 162, 161, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 162, 3, 2, 2, 2, 164, 165, 3, 2, 2, 2, 165, 166, 3, 2, 2, 2, 166, 167, 8, 10, 4, 2, 167, 24, 3, 2, 2, 2, 168, 169, 7, 38, 2, 2, 169, 170, 3, 2, 2, 2, 170, 171, 8, 11, 4, 2, 171, 26, 3, 2, 2, 2, 172, 173, 7, 38, 2, 2, 173, 174, 7, 125, 2, 2, 174, 176, 3, 2, 2, 2, 175, 177, 5, 63, 30, 2, 176, 175, 3, 2, 2, 2, 176, 177, 3, 2, 2, 2, 177, 178, 3, 2, 2, 2, 178, 179, 8, 12, 3, 2, 179, 28, 3, 2, 2, 2, 180, 182, 7, 125, 2, 2, 181, 183, 5, 63, 30, 2, 182, 181, 3, 2, 2, 2, 182, 183, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, 184, 185, 8, 13, 5, 2, 185, 30, 3, 2, 2, 2, 186, 188, 5, 63, 30, 2, 187, 186, 3, 2, 2, 2, 187, 188, 3, 2, 2, 2, 188, 189, 3, 2, 2, 2, 189, 190, 7, 127, 2, 2, 190, 191, 3, 2, 2, 2, 191, 192, 8, 14, 6, 2, 192, 32, 3, 2, 2, 2, 193, 194, 7, 41, 2, 2, 194, 195, 3, 2, 2, 2, 195, 196, 8, 15, 7, 2, 196, 34, 3, 2, 2, 2, 197, 198, 7, 36, 2, 2, 198, 199, 3, 2, 2, 2, 199, 200, 8, 16, 8, 2, 200, 36, 3, 2, 2, 2, 201, 203, 5, 63, 30, 2, 202, 201, 3, 2, 2, 2, 202, 203, 3, 2, 2, 2, 203, 204, 3, 2, 2, 2, 204, 206, 7, 46, 2, 2, 205, 207, 5, 63, 30, 2, 206, 205, 3, 2, 2, 2, 206, 207, 3, 2, 2, 2, 207, 38, 3, 2, 2, 2, 208, 210, 7, 93, 2, 2, 209, 211, 5, 63, 30, 2, 210, 209, 3, 2, 2, 2, 210, 211, 3, 2, 2, 2, 211, 40, 3, 2, 2, 2, 212, 214, 5, 63, 30, 2, 213, 212, 3, 2, 2, 2, 213, 214, 3, 2, 2, 2, 214, 215, 3, 2, 2, 2, 215, 216, 7, 95, 2, 2, 216, 42, 3, 2, 2, 2, 217, 219, 5, 63, 30, 2, 218, 217, 3, 2, 2, 2, 218, 219, 3, 2, 2, 2, 219, 220, 3, 2, 2, 2, 220, 222, 7, 60, 2, 2, 221, 223, 5, 63, 30, 2, 222, 221, 3, 2, 2, 2, 222, 223, 3, 2, 2, 2, 223, 44, 3, 2, 2, 2, 224, 225, 5, 11, 4, 2, 225, 226, 7, 48, 2, 2, 226, 242, 3, 2, 2, 2, 227, 229, 5, 11, 4, 2, 228, 227, 3, 2, 2, 2, 228, 229, 3, 2, 2, 2, 229, 230, 3, 2, 2, 2, 230, 231, 7, 48, 2, 2, 231, 238, 5, 9, 3, 2, 232, 234, 7, 97, 2, 2, 233, 232, 3, 2, 2, 2, 233, 234, 3, 2, 2, 2, 234, 235, 3, 2, 2, 2, 235, 237, 5, 9, 3, 2, 236, 233, 3, 2, 2, 2, 237, 240, 3, 2, 2, 2, 238, 236, 3, 2, 2, 2, 238, 239, 3, 2, 2, 2, 239, 242, 3, 2, 2, 2, 240, 238, 3, 2, 2, 2, 241, 224, 3, 2, 2, 2, 241, 228, 3, 2, 2, 2, 242, 46, 3, 2, 2, 2, 243, 246, 5, 11, 4, 2, 244, 246, 5, 45, 21, 2, 245, 243, 3, 2, 2, 2, 245, 244, 3, 2, 2, 2, 246, 247, 3, 2, 2, 2, 247, 249, 9, 7, 2, 2, 248, 250, 9, 8, 2, 2, 249, 248, 3, 2, 2, 2, 249, 250, 3, 2, 2, 2, 250, 251, 3, 2, 2, 2, 251, 258, 5, 9, 3, 2, 252, 254, 7, 97, 2, 2, 253, 252, 3, 2, 2, 2, 253, 254, 3, 2, 2, 2, 254, 255, 3, 2, 2, 2, 255, 257, 5, 9, 3, 2, 256, 253, 3, 2, 2, 2, 257, 260, 3, 2, 2, 2, 258, 256, 3, 2, 2, 2, 258, 259, 3, 2, 2, 2, 259, 48, 3, 2, 2, 2, 260, 258, 3, 2, 2, 2, 261, 263, 9, 8, 2, 2, 262, 261, 3, 2, 2, 2, 262, 263, 3, 2, 2, 2, 263, 272, 3, 2, 2, 2, 264, 273, 5, 45, 21, 2, 265, 273, 5, 47, 22, 2, 266, 267, 9, 9, 2, 2, 267, 268, 9, 10, 2, 2, 268, 273, 9, 11, 2, 2, 269, 270, 9, 10, 2, 2, 270, 271, 9, 12, 2, 2, 271, 273, 9, 10, 2, 2, 272, 264, 3, 2, 2, 2, 272, 265, 3, 2, 2, 2, 272, 266, 3, 2, 2, 2, 272, 269, 3, 2, 2, 2, 273, 50, 3, 2, 2, 2, 274, 276, 9, 8, 2, 2, 275, 274, 3, 2, 2, 2, 275, 276, 3, 2, 2, 2, 276, 277, 3, 2, 2, 2, 277, 278, 5, 11, 4, 2, 278, 52, 3, 2, 2, 2, 279, 280, 9, 13, 2, 2, 280, 281, 9, 14, 2, 2, 281, 282, 9, 15, 2, 2, 282, 289, 9, 7, 2, 2, 283, 284, 9, 11, 2, 2, 284, 285, 9, 12, 2, 2, 285, 286, 9, 16, 2, 2, 286, 287, 9, 17, 2, 2, 287, 289, 9, 7, 2, 2, 288, 279, 3, 2, 2, 2, 288, 283, 3, 2, 2, 2, 289, 54, 3, 2, 2, 2, 290, 291, 9, 10, 2, 2, 291, 292, 9, 15, 2, 2, 292, 293, 9, 16, 2, 2, 293, 294, 9, 16, 2, 2, 294, 56, 3, 2, 2, 2, 295, 296, 9, 18, 2, 2, 296, 58, 3, 2, 2, 2, 297, 300, 5, 7, 2, 2, 298, 300, 7, 97, 2, 2, 299, 297, 3, 2, 2, 2, 299, 298, 3, 2, 2, 2, 300, 306, 3, 2, 2, 2, 301, 305, 5, 7, 2, 2, 302, 305, 5, 9, 3, 2, 303, 305, 9, 19, 2, 2, 304, 301, 3, 2, 2, 2, 304, 302, 3, 2, 2, 2, 304, 303, 3, 2, 2, 2, 305, 308, 3, 2, 2, 2, 306, 304, 3, 2, 2, 2, 306, 307, 3, 2, 2, 2, 307, 60, 3, 2, 2, 2, 308, 306, 3, 2, 2, 2, 309, 333, 5, 13, 5, 2, 310, 311, 7, 94, 2, 2, 311, 333, 7, 42, 2, 2, 312, 313, 7, 94, 2, 2, 313, 333, 7, 43, 2, 2, 314, 315, 7, 94, 2, 2, 315, 333, 7, 93, 2, 2, 316, 317, 7, 94, 2, 2, 317, 333, 7, 95, 2, 2, 318, 319, 7, 94, 2, 2, 319, 333, 7, 125, 2, 2, 320, 321, 7, 94, 2, 2, 321, 333, 7, 127, 2, 2, 322, 323, 7, 94, 2, 2, 323, 333, 7, 60, 2, 2, 324, 325, 7, 94, 2, 2, 325, 333, 7, 63, 2, 2, 326, 327, 7, 94, 2, 2, 327, 333, 7, 46, 2, 2, 328, 329, 7, 94, 2, 2, 329, 333, 7, 34, 2, 2, 330, 331, 7, 94, 2, 2, 331, 333, 7, 11, 2, 2, 332, 309, 3, 2, 2, 2, 332, 310, 3, 2, 2, 2, 332, 312, 3, 2, 2, 2, 332, 314, 3, 2, 2, 2, 332, 316, 3, 2, 2, 2, 332, 318, 3, 2, 2, 2, 332, 320, 3, 2, 2, 2, 332, 322, 3, 2, 2, 2, 332, 324, 3, 2, 2, 2, 332, 326, 3, 2, 2, 2, 332, 328, 3, 2, 2, 2, 332, 330, 3, 2, 2, 2, 333, 334, 3, 2, 2, 2, 334, 332, 3, 2, 2, 2, 334, 335, 3, 2, 2, 2, 335, 62, 3, 2, 2, 2, 336, 338, 9, 20, 2, 2, 337, 336, 3, 2, 2, 2, 338, 339, 3, 2, 2, 2, 339, 337, 3, 2, 2, 2, 339, 340, 3, 2, 2, 2, 340, 64, 3, 2, 2, 2, 341, 343, 5, 27, 12, 2, 342, 344, 5, 63, 30, 2, 343, 342, 3, 2, 2, 2, 343, 344, 3, 2, 2, 2, 344, 345, 3, 2, 2, 2, 345, 346, 8, 31, 2, 2, 346, 347, 8, 31, 3, 2, 347, 66, 3, 2, 2, 2, 348, 350, 5, 63, 30, 2, 349, 348, 3, 2, 2, 2, 349, 350, 3, 2, 2, 2, 350, 351, 3, 2, 2, 2, 351, 353, 7, 60, 2, 2, 352, 354, 5, 63, 30, 2, 353, 352, 3, 2, 2, 2, 353, 354, 3, 2, 2, 2, 354, 355, 3, 2, 2, 2, 355, 356, 8, 32, 9, 2, 356, 357, 8, 32, 10, 2, 357, 68, 3, 2, 2, 2, 358, 360, 5, 63, 30, 2, 359, 358, 3, 2, 2, 2, 359, 360, 3, 2, 2, 2, 360, 361, 3, 2, 2, 2, 361, 362, 7, 127, 2, 2, 362, 363, 3, 2, 2, 2, 363, 364, 8, 33, 6, 2, 364, 70, 3, 2, 2, 2, 365, 366, 7, 48, 2, 2, 366, 72, 3, 2, 2, 2, 367, 368, 7, 93, 2, 2, 368, 369, 3, 2, 2, 2, 369, 370, 8, 35, 11, 2, 370, 74, 3, 2, 2, 2, 371, 372, 7, 95, 2, 2, 372, 373, 3, 2, 2, 2, 373, 374, 8, 36, 12, 2, 374, 76, 3, 2, 2, 2, 375, 376, 5, 59, 28, 2, 376, 377, 3, 2, 2, 2, 377, 378, 8, 37, 13, 2, 378, 78, 3, 2, 2, 2, 379, 381, 10, 21, 2, 2, 380, 379, 3, 2, 2, 2, 381, 382, 3, 2, 2, 2, 382, 380, 3, 2, 2, 2, 382, 383, 3, 2, 2, 2, 383, 80, 3, 2, 2, 2, 384, 385, 5, 27, 12, 2, 385, 386, 3, 2, 2, 2, 386, 387, 8, 39, 2, 2, 387, 388, 8, 39, 3, 2, 388, 82, 3, 2, 2, 2, 389, 390, 7, 41, 2, 2, 390, 391, 3, 2, 2, 2, 391, 392, 8, 40, 6, 2, 392, 84, 3, 2, 2, 2, 393, 395, 10, 22, 2, 2, 394, 393, 3, 2, 2, 2, 395, 398, 3, 2, 2, 2, 396, 394, 3, 2, 2, 2, 396, 397, 3, 2, 2, 2, 397, 399, 3, 2, 2, 2, 398, 396, 3, 2, 2, 2, 399, 400, 10, 23, 2, 2, 400, 401, 3, 2, 2, 2, 401, 402, 8, 41, 4, 2, 402, 86, 3, 2, 2, 2, 403, 404, 5, 19, 8, 2, 404, 405, 3, 2, 2, 2, 405, 406, 8, 42, 14, 2, 406, 88, 3, 2, 2, 2, 407, 409, 5, 13, 5, 2, 408, 407, 3, 2, 2, 2, 409, 412, 3, 2, 2, 2, 410, 408, 3, 2, 2, 2, 410, 411, 3, 2, 2, 2, 411, 413, 3, 2, 2, 2, 412, 410, 3, 2, 2, 2, 413, 414, 7, 94, 2, 2, 414, 415, 7, 41, 2, 2, 415, 416, 3, 2, 2, 2, 416, 417, 8, 43, 15, 2, 417, 90, 3, 2, 2, 2, 418, 420, 5, 13, 5, 2, 419, 418, 3, 2, 2, 2, 420, 421, 3, 2, 2, 2, 421, 419, 3, 2, 2, 2, 421, 422, 3, 2, 2, 2, 422, 92, 3, 2, 2, 2, 423, 425, 7, 94, 2, 2, 424, 423, 3, 2, 2, 2, 425, 426, 3, 2, 2, 2, 426, 424, 3, 2, 2, 2, 426, 427, 3, 2, 2, 2, 427, 428, 3, 2, 2, 2, 428, 429, 8, 45, 4, 2, 429, 94, 3, 2, 2, 2, 430, 431, 7, 38, 2, 2, 431, 432, 3, 2, 2, 2, 432, 433, 8, 46, 4, 2, 433, 96, 3, 2, 2, 2, 434, 435, 5, 27, 12, 2, 435, 436, 3, 2, 2, 2, 436, 437, 8, 47, 2, 2, 437, 438, 8, 47, 3, 2, 438, 98, 3, 2, 2, 2, 439, 440, 7, 36, 2, 2, 440, 441, 3, 2, 2, 2, 441, 442, 8, 48, 16, 2, 442, 443, 8, 48, 6, 2, 443, 100, 3, 2, 2, 2, 444, 446, 10, 24, 2, 2, 445, 444, 3, 2, 2, 2, 446, 449, 3, 2, 2, 2, 447, 445, 3, 2, 2, 2, 447, 448, 3, 2, 2, 2, 448, 450, 3, 2, 2, 2, 449, 447, 3, 2, 2, 2, 450, 451, 10, 25, 2, 2, 451, 452, 3, 2, 2, 2, 452, 453, 8, 49, 4, 2, 453, 102, 3, 2, 2, 2, 454, 455, 5, 19, 8, 2, 455, 456, 3, 2, 2, 2, 456, 457, 8, 50, 14, 2, 457, 104, 3, 2, 2, 2, 458, 460, 5, 13, 5, 2, 459, 458, 3, 2, 2, 2, 460, 463, 3, 2, 2, 2, 461, 459, 3, 2, 2, 2, 461, 462, 3, 2, 2, 2, 462, 464, 3, 2, 2, 2, 463, 461, 3, 2, 2, 2, 464, 465, 7, 94, 2, 2, 465, 466, 7, 36, 2, 2, 466, 467, 3, 2, 2, 2, 467, 468, 8, 51, 15, 2, 468, 106, 3, 2, 2, 2, 469, 471, 5, 13, 5, 2, 470, 469, 3, 2, 2, 2, 471, 472, 3, 2, 2, 2, 472, 470, 3, 2, 2, 2, 472, 473, 3, 2, 2, 2, 473, 474, 3, 2, 2, 2, 474, 475, 8, 52, 17, 2, 475, 108, 3, 2, 2, 2, 476, 478, 7, 94, 2, 2, 477, 476, 3, 2, 2, 2, 478, 479, 3, 2, 2, 2, 479, 477, 3, 2, 2, 2, 479, 480, 3, 2, 2, 2, 480, 481, 3, 2, 2, 2, 481, 482, 8, 53, 4, 2, 482, 110, 3, 2, 2, 2, 483, 484, 7, 38, 2, 2, 484, 485, 3, 2, 2, 2, 485, 486, 8, 54, 4, 2, 486, 112, 3, 2, 2, 2, 54, 2, 3, 4, 5, 6, 120, 125, 128, 141, 149, 159, 164, 176, 182, 187, 202, 206, 210, 213, 218, 222, 228, 233, 238, 241, 245, 249, 253, 258, 262, 272, 275, 288, 299, 304, 306, 332, 334, 339, 343, 349, 353, 359, 382, 396, 410, 421, 426, 447, 461, 472, 479, 18, 9, 6, 2, 7, 4, 2, 9, 3, 2, 7, 3, 2, 6, 2, 2, 7, 5, 2, 7, 6, 2, 9, 14, 2, 4, 3, 2, 9, 12, 2, 9, 13, 2, 9, 20, 2, 9, 4, 2, 9, 21, 2, 9, 26, 2, 9, 27, 2] \ No newline at end of file diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.py b/omegaconf/grammar/gen/OmegaConfGrammarLexer.py deleted file mode 100644 index 762a09e71..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarLexer.py +++ /dev/null @@ -1,404 +0,0 @@ -# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarLexer.g4 by ANTLR 4.9.3 -from antlr4 import * -from io import StringIO -import sys - -if sys.version_info[1] > 5: - from typing import TextIO -else: - from typing.io import TextIO - - -def serializedATN(): - with StringIO() as buf: - buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\36") - buf.write("\u01e7\b\1\b\1\b\1\b\1\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5") - buf.write("\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13") - buf.write("\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t") - buf.write("\21\4\22\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26") - buf.write("\4\27\t\27\4\30\t\30\4\31\t\31\4\32\t\32\4\33\t\33\4\34") - buf.write('\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!\t!\4"\t') - buf.write("\"\4#\t#\4$\t$\4%\t%\4&\t&\4'\t'\4(\t(\4)\t)\4*\t*\4") - buf.write("+\t+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62") - buf.write("\t\62\4\63\t\63\4\64\t\64\4\65\t\65\4\66\t\66\3\2\3\2") - buf.write("\3\3\3\3\3\4\3\4\3\4\5\4y\n\4\3\4\7\4|\n\4\f\4\16\4\177") - buf.write("\13\4\5\4\u0081\n\4\3\5\3\5\3\5\3\6\3\6\3\6\3\6\3\6\3") - buf.write("\7\7\7\u008c\n\7\f\7\16\7\u008f\13\7\3\7\3\7\3\b\7\b\u0094") - buf.write("\n\b\f\b\16\b\u0097\13\b\3\b\3\b\3\b\3\b\3\t\6\t\u009e") - buf.write("\n\t\r\t\16\t\u009f\3\n\6\n\u00a3\n\n\r\n\16\n\u00a4\3") - buf.write("\n\3\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\3\f\5\f\u00b1\n") - buf.write("\f\3\f\3\f\3\r\3\r\5\r\u00b7\n\r\3\r\3\r\3\16\5\16\u00bc") - buf.write("\n\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\20\3\20") - buf.write("\3\20\3\20\3\21\5\21\u00cb\n\21\3\21\3\21\5\21\u00cf\n") - buf.write("\21\3\22\3\22\5\22\u00d3\n\22\3\23\5\23\u00d6\n\23\3\23") - buf.write("\3\23\3\24\5\24\u00db\n\24\3\24\3\24\5\24\u00df\n\24\3") - buf.write("\25\3\25\3\25\3\25\5\25\u00e5\n\25\3\25\3\25\3\25\5\25") - buf.write("\u00ea\n\25\3\25\7\25\u00ed\n\25\f\25\16\25\u00f0\13\25") - buf.write("\5\25\u00f2\n\25\3\26\3\26\5\26\u00f6\n\26\3\26\3\26\5") - buf.write("\26\u00fa\n\26\3\26\3\26\5\26\u00fe\n\26\3\26\7\26\u0101") - buf.write("\n\26\f\26\16\26\u0104\13\26\3\27\5\27\u0107\n\27\3\27") - buf.write("\3\27\3\27\3\27\3\27\3\27\3\27\3\27\5\27\u0111\n\27\3") - buf.write("\30\5\30\u0114\n\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31") - buf.write("\3\31\3\31\3\31\3\31\5\31\u0121\n\31\3\32\3\32\3\32\3") - buf.write("\32\3\32\3\33\3\33\3\34\3\34\5\34\u012c\n\34\3\34\3\34") - buf.write("\3\34\7\34\u0131\n\34\f\34\16\34\u0134\13\34\3\35\3\35") - buf.write("\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35") - buf.write("\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\6\35") - buf.write("\u014d\n\35\r\35\16\35\u014e\3\36\6\36\u0152\n\36\r\36") - buf.write("\16\36\u0153\3\37\3\37\5\37\u0158\n\37\3\37\3\37\3\37") - buf.write("\3 \5 \u015e\n \3 \3 \5 \u0162\n \3 \3 \3 \3!\5!\u0168") - buf.write('\n!\3!\3!\3!\3!\3"\3"\3#\3#\3#\3#\3$\3$\3$\3$\3%\3%') - buf.write("\3%\3%\3&\6&\u017d\n&\r&\16&\u017e\3'\3'\3'\3'\3'") - buf.write("\3(\3(\3(\3(\3)\7)\u018b\n)\f)\16)\u018e\13)\3)\3)\3)") - buf.write("\3)\3*\3*\3*\3*\3+\7+\u0199\n+\f+\16+\u019c\13+\3+\3+") - buf.write("\3+\3+\3+\3,\6,\u01a4\n,\r,\16,\u01a5\3-\6-\u01a9\n-\r") - buf.write("-\16-\u01aa\3-\3-\3.\3.\3.\3.\3/\3/\3/\3/\3/\3\60\3\60") - buf.write("\3\60\3\60\3\60\3\61\7\61\u01be\n\61\f\61\16\61\u01c1") - buf.write("\13\61\3\61\3\61\3\61\3\61\3\62\3\62\3\62\3\62\3\63\7") - buf.write("\63\u01cc\n\63\f\63\16\63\u01cf\13\63\3\63\3\63\3\63\3") - buf.write("\63\3\63\3\64\6\64\u01d7\n\64\r\64\16\64\u01d8\3\64\3") - buf.write("\64\3\65\6\65\u01de\n\65\r\65\16\65\u01df\3\65\3\65\3") - buf.write("\66\3\66\3\66\3\66\2\2\67\7\2\t\2\13\2\r\2\17\2\21\3\23") - buf.write("\4\25\5\27\2\31\34\33\6\35\7\37\b!\t#\n%\13'\f)\r+\16") - buf.write("-\2/\2\61\17\63\20\65\21\67\229\23;\24=\25?\26A\2C\2E") - buf.write("\27G\30I\35K\36M\2O\31Q\2S\32U\2W\2Y\2[\33]\2_\2a\2c\2") - buf.write("e\2g\2i\2k\2m\2o\2\7\2\3\4\5\6\32\4\2C\\c|\3\2\62;\3\2") - buf.write("\63;\3\2&&\4\2&&^^\4\2GGgg\4\2--//\4\2KKkk\4\2PPpp\4\2") - buf.write("HHhh\4\2CCcc\4\2VVvv\4\2TTtt\4\2WWww\4\2NNnn\4\2UUuu\b") - buf.write('\2&\',-/\61AB^^~~\4\2//aa\4\2\13\13""\13\2\13\13""') - buf.write("$$)+\60\60<<]_}}\177\177\4\2&&))\5\2&&))^^\4\2$$&&\5\2") - buf.write("$$&&^^\2\u0218\2\17\3\2\2\2\2\21\3\2\2\2\2\23\3\2\2\2") - buf.write("\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3\2\2\2\3\33\3\2\2\2\3") - buf.write("\35\3\2\2\2\3\37\3\2\2\2\3!\3\2\2\2\3#\3\2\2\2\3%\3\2") - buf.write("\2\2\3'\3\2\2\2\3)\3\2\2\2\3+\3\2\2\2\3\61\3\2\2\2\3") - buf.write("\63\3\2\2\2\3\65\3\2\2\2\3\67\3\2\2\2\39\3\2\2\2\3;\3") - buf.write("\2\2\2\3=\3\2\2\2\3?\3\2\2\2\4A\3\2\2\2\4C\3\2\2\2\4E") - buf.write("\3\2\2\2\4G\3\2\2\2\4I\3\2\2\2\4K\3\2\2\2\4M\3\2\2\2\4") - buf.write("O\3\2\2\2\5Q\3\2\2\2\5S\3\2\2\2\5U\3\2\2\2\5W\3\2\2\2") - buf.write("\5Y\3\2\2\2\5[\3\2\2\2\5]\3\2\2\2\5_\3\2\2\2\6a\3\2\2") - buf.write("\2\6c\3\2\2\2\6e\3\2\2\2\6g\3\2\2\2\6i\3\2\2\2\6k\3\2") - buf.write("\2\2\6m\3\2\2\2\6o\3\2\2\2\7q\3\2\2\2\ts\3\2\2\2\13\u0080") - buf.write("\3\2\2\2\r\u0082\3\2\2\2\17\u0085\3\2\2\2\21\u008d\3\2") - buf.write("\2\2\23\u0095\3\2\2\2\25\u009d\3\2\2\2\27\u00a2\3\2\2") - buf.write("\2\31\u00a8\3\2\2\2\33\u00ac\3\2\2\2\35\u00b4\3\2\2\2") - buf.write("\37\u00bb\3\2\2\2!\u00c1\3\2\2\2#\u00c5\3\2\2\2%\u00ca") - buf.write("\3\2\2\2'\u00d0\3\2\2\2)\u00d5\3\2\2\2+\u00da\3\2\2\2") - buf.write("-\u00f1\3\2\2\2/\u00f5\3\2\2\2\61\u0106\3\2\2\2\63\u0113") - buf.write("\3\2\2\2\65\u0120\3\2\2\2\67\u0122\3\2\2\29\u0127\3\2") - buf.write("\2\2;\u012b\3\2\2\2=\u014c\3\2\2\2?\u0151\3\2\2\2A\u0155") - buf.write("\3\2\2\2C\u015d\3\2\2\2E\u0167\3\2\2\2G\u016d\3\2\2\2") - buf.write("I\u016f\3\2\2\2K\u0173\3\2\2\2M\u0177\3\2\2\2O\u017c\3") - buf.write("\2\2\2Q\u0180\3\2\2\2S\u0185\3\2\2\2U\u018c\3\2\2\2W\u0193") - buf.write("\3\2\2\2Y\u019a\3\2\2\2[\u01a3\3\2\2\2]\u01a8\3\2\2\2") - buf.write("_\u01ae\3\2\2\2a\u01b2\3\2\2\2c\u01b7\3\2\2\2e\u01bf\3") - buf.write("\2\2\2g\u01c6\3\2\2\2i\u01cd\3\2\2\2k\u01d6\3\2\2\2m\u01dd") - buf.write("\3\2\2\2o\u01e3\3\2\2\2qr\t\2\2\2r\b\3\2\2\2st\t\3\2\2") - buf.write("t\n\3\2\2\2u\u0081\7\62\2\2v}\t\4\2\2wy\7a\2\2xw\3\2\2") - buf.write("\2xy\3\2\2\2yz\3\2\2\2z|\5\t\3\2{x\3\2\2\2|\177\3\2\2") - buf.write("\2}{\3\2\2\2}~\3\2\2\2~\u0081\3\2\2\2\177}\3\2\2\2\u0080") - buf.write("u\3\2\2\2\u0080v\3\2\2\2\u0081\f\3\2\2\2\u0082\u0083\7") - buf.write("^\2\2\u0083\u0084\7^\2\2\u0084\16\3\2\2\2\u0085\u0086") - buf.write("\5\33\f\2\u0086\u0087\3\2\2\2\u0087\u0088\b\6\2\2\u0088") - buf.write("\u0089\b\6\3\2\u0089\20\3\2\2\2\u008a\u008c\n\5\2\2\u008b") - buf.write("\u008a\3\2\2\2\u008c\u008f\3\2\2\2\u008d\u008b\3\2\2\2") - buf.write("\u008d\u008e\3\2\2\2\u008e\u0090\3\2\2\2\u008f\u008d\3") - buf.write("\2\2\2\u0090\u0091\n\6\2\2\u0091\22\3\2\2\2\u0092\u0094") - buf.write("\5\r\5\2\u0093\u0092\3\2\2\2\u0094\u0097\3\2\2\2\u0095") - buf.write("\u0093\3\2\2\2\u0095\u0096\3\2\2\2\u0096\u0098\3\2\2\2") - buf.write("\u0097\u0095\3\2\2\2\u0098\u0099\7^\2\2\u0099\u009a\7") - buf.write("&\2\2\u009a\u009b\7}\2\2\u009b\24\3\2\2\2\u009c\u009e") - buf.write("\5\r\5\2\u009d\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f") - buf.write("\u009d\3\2\2\2\u009f\u00a0\3\2\2\2\u00a0\26\3\2\2\2\u00a1") - buf.write("\u00a3\7^\2\2\u00a2\u00a1\3\2\2\2\u00a3\u00a4\3\2\2\2") - buf.write("\u00a4\u00a2\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5\u00a6\3") - buf.write("\2\2\2\u00a6\u00a7\b\n\4\2\u00a7\30\3\2\2\2\u00a8\u00a9") - buf.write("\7&\2\2\u00a9\u00aa\3\2\2\2\u00aa\u00ab\b\13\4\2\u00ab") - buf.write("\32\3\2\2\2\u00ac\u00ad\7&\2\2\u00ad\u00ae\7}\2\2\u00ae") - buf.write("\u00b0\3\2\2\2\u00af\u00b1\5?\36\2\u00b0\u00af\3\2\2\2") - buf.write("\u00b0\u00b1\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b3\b") - buf.write("\f\3\2\u00b3\34\3\2\2\2\u00b4\u00b6\7}\2\2\u00b5\u00b7") - buf.write("\5?\36\2\u00b6\u00b5\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7") - buf.write("\u00b8\3\2\2\2\u00b8\u00b9\b\r\5\2\u00b9\36\3\2\2\2\u00ba") - buf.write("\u00bc\5?\36\2\u00bb\u00ba\3\2\2\2\u00bb\u00bc\3\2\2\2") - buf.write("\u00bc\u00bd\3\2\2\2\u00bd\u00be\7\177\2\2\u00be\u00bf") - buf.write("\3\2\2\2\u00bf\u00c0\b\16\6\2\u00c0 \3\2\2\2\u00c1\u00c2") - buf.write("\7)\2\2\u00c2\u00c3\3\2\2\2\u00c3\u00c4\b\17\7\2\u00c4") - buf.write('"\3\2\2\2\u00c5\u00c6\7$\2\2\u00c6\u00c7\3\2\2\2\u00c7') - buf.write("\u00c8\b\20\b\2\u00c8$\3\2\2\2\u00c9\u00cb\5?\36\2\u00ca") - buf.write("\u00c9\3\2\2\2\u00ca\u00cb\3\2\2\2\u00cb\u00cc\3\2\2\2") - buf.write("\u00cc\u00ce\7.\2\2\u00cd\u00cf\5?\36\2\u00ce\u00cd\3") - buf.write("\2\2\2\u00ce\u00cf\3\2\2\2\u00cf&\3\2\2\2\u00d0\u00d2") - buf.write("\7]\2\2\u00d1\u00d3\5?\36\2\u00d2\u00d1\3\2\2\2\u00d2") - buf.write("\u00d3\3\2\2\2\u00d3(\3\2\2\2\u00d4\u00d6\5?\36\2\u00d5") - buf.write("\u00d4\3\2\2\2\u00d5\u00d6\3\2\2\2\u00d6\u00d7\3\2\2\2") - buf.write("\u00d7\u00d8\7_\2\2\u00d8*\3\2\2\2\u00d9\u00db\5?\36\2") - buf.write("\u00da\u00d9\3\2\2\2\u00da\u00db\3\2\2\2\u00db\u00dc\3") - buf.write("\2\2\2\u00dc\u00de\7<\2\2\u00dd\u00df\5?\36\2\u00de\u00dd") - buf.write("\3\2\2\2\u00de\u00df\3\2\2\2\u00df,\3\2\2\2\u00e0\u00e1") - buf.write("\5\13\4\2\u00e1\u00e2\7\60\2\2\u00e2\u00f2\3\2\2\2\u00e3") - buf.write("\u00e5\5\13\4\2\u00e4\u00e3\3\2\2\2\u00e4\u00e5\3\2\2") - buf.write("\2\u00e5\u00e6\3\2\2\2\u00e6\u00e7\7\60\2\2\u00e7\u00ee") - buf.write("\5\t\3\2\u00e8\u00ea\7a\2\2\u00e9\u00e8\3\2\2\2\u00e9") - buf.write("\u00ea\3\2\2\2\u00ea\u00eb\3\2\2\2\u00eb\u00ed\5\t\3\2") - buf.write("\u00ec\u00e9\3\2\2\2\u00ed\u00f0\3\2\2\2\u00ee\u00ec\3") - buf.write("\2\2\2\u00ee\u00ef\3\2\2\2\u00ef\u00f2\3\2\2\2\u00f0\u00ee") - buf.write("\3\2\2\2\u00f1\u00e0\3\2\2\2\u00f1\u00e4\3\2\2\2\u00f2") - buf.write(".\3\2\2\2\u00f3\u00f6\5\13\4\2\u00f4\u00f6\5-\25\2\u00f5") - buf.write("\u00f3\3\2\2\2\u00f5\u00f4\3\2\2\2\u00f6\u00f7\3\2\2\2") - buf.write("\u00f7\u00f9\t\7\2\2\u00f8\u00fa\t\b\2\2\u00f9\u00f8\3") - buf.write("\2\2\2\u00f9\u00fa\3\2\2\2\u00fa\u00fb\3\2\2\2\u00fb\u0102") - buf.write("\5\t\3\2\u00fc\u00fe\7a\2\2\u00fd\u00fc\3\2\2\2\u00fd") - buf.write("\u00fe\3\2\2\2\u00fe\u00ff\3\2\2\2\u00ff\u0101\5\t\3\2") - buf.write("\u0100\u00fd\3\2\2\2\u0101\u0104\3\2\2\2\u0102\u0100\3") - buf.write("\2\2\2\u0102\u0103\3\2\2\2\u0103\60\3\2\2\2\u0104\u0102") - buf.write("\3\2\2\2\u0105\u0107\t\b\2\2\u0106\u0105\3\2\2\2\u0106") - buf.write("\u0107\3\2\2\2\u0107\u0110\3\2\2\2\u0108\u0111\5-\25\2") - buf.write("\u0109\u0111\5/\26\2\u010a\u010b\t\t\2\2\u010b\u010c\t") - buf.write("\n\2\2\u010c\u0111\t\13\2\2\u010d\u010e\t\n\2\2\u010e") - buf.write("\u010f\t\f\2\2\u010f\u0111\t\n\2\2\u0110\u0108\3\2\2\2") - buf.write("\u0110\u0109\3\2\2\2\u0110\u010a\3\2\2\2\u0110\u010d\3") - buf.write("\2\2\2\u0111\62\3\2\2\2\u0112\u0114\t\b\2\2\u0113\u0112") - buf.write("\3\2\2\2\u0113\u0114\3\2\2\2\u0114\u0115\3\2\2\2\u0115") - buf.write("\u0116\5\13\4\2\u0116\64\3\2\2\2\u0117\u0118\t\r\2\2\u0118") - buf.write("\u0119\t\16\2\2\u0119\u011a\t\17\2\2\u011a\u0121\t\7\2") - buf.write("\2\u011b\u011c\t\13\2\2\u011c\u011d\t\f\2\2\u011d\u011e") - buf.write("\t\20\2\2\u011e\u011f\t\21\2\2\u011f\u0121\t\7\2\2\u0120") - buf.write("\u0117\3\2\2\2\u0120\u011b\3\2\2\2\u0121\66\3\2\2\2\u0122") - buf.write("\u0123\t\n\2\2\u0123\u0124\t\17\2\2\u0124\u0125\t\20\2") - buf.write("\2\u0125\u0126\t\20\2\2\u01268\3\2\2\2\u0127\u0128\t\22") - buf.write("\2\2\u0128:\3\2\2\2\u0129\u012c\5\7\2\2\u012a\u012c\7") - buf.write("a\2\2\u012b\u0129\3\2\2\2\u012b\u012a\3\2\2\2\u012c\u0132") - buf.write("\3\2\2\2\u012d\u0131\5\7\2\2\u012e\u0131\5\t\3\2\u012f") - buf.write("\u0131\t\23\2\2\u0130\u012d\3\2\2\2\u0130\u012e\3\2\2") - buf.write("\2\u0130\u012f\3\2\2\2\u0131\u0134\3\2\2\2\u0132\u0130") - buf.write("\3\2\2\2\u0132\u0133\3\2\2\2\u0133<\3\2\2\2\u0134\u0132") - buf.write("\3\2\2\2\u0135\u014d\5\r\5\2\u0136\u0137\7^\2\2\u0137") - buf.write("\u014d\7*\2\2\u0138\u0139\7^\2\2\u0139\u014d\7+\2\2\u013a") - buf.write("\u013b\7^\2\2\u013b\u014d\7]\2\2\u013c\u013d\7^\2\2\u013d") - buf.write("\u014d\7_\2\2\u013e\u013f\7^\2\2\u013f\u014d\7}\2\2\u0140") - buf.write("\u0141\7^\2\2\u0141\u014d\7\177\2\2\u0142\u0143\7^\2\2") - buf.write("\u0143\u014d\7<\2\2\u0144\u0145\7^\2\2\u0145\u014d\7?") - buf.write("\2\2\u0146\u0147\7^\2\2\u0147\u014d\7.\2\2\u0148\u0149") - buf.write('\7^\2\2\u0149\u014d\7"\2\2\u014a\u014b\7^\2\2\u014b\u014d') - buf.write("\7\13\2\2\u014c\u0135\3\2\2\2\u014c\u0136\3\2\2\2\u014c") - buf.write("\u0138\3\2\2\2\u014c\u013a\3\2\2\2\u014c\u013c\3\2\2\2") - buf.write("\u014c\u013e\3\2\2\2\u014c\u0140\3\2\2\2\u014c\u0142\3") - buf.write("\2\2\2\u014c\u0144\3\2\2\2\u014c\u0146\3\2\2\2\u014c\u0148") - buf.write("\3\2\2\2\u014c\u014a\3\2\2\2\u014d\u014e\3\2\2\2\u014e") - buf.write("\u014c\3\2\2\2\u014e\u014f\3\2\2\2\u014f>\3\2\2\2\u0150") - buf.write("\u0152\t\24\2\2\u0151\u0150\3\2\2\2\u0152\u0153\3\2\2") - buf.write("\2\u0153\u0151\3\2\2\2\u0153\u0154\3\2\2\2\u0154@\3\2") - buf.write("\2\2\u0155\u0157\5\33\f\2\u0156\u0158\5?\36\2\u0157\u0156") - buf.write("\3\2\2\2\u0157\u0158\3\2\2\2\u0158\u0159\3\2\2\2\u0159") - buf.write("\u015a\b\37\2\2\u015a\u015b\b\37\3\2\u015bB\3\2\2\2\u015c") - buf.write("\u015e\5?\36\2\u015d\u015c\3\2\2\2\u015d\u015e\3\2\2\2") - buf.write("\u015e\u015f\3\2\2\2\u015f\u0161\7<\2\2\u0160\u0162\5") - buf.write("?\36\2\u0161\u0160\3\2\2\2\u0161\u0162\3\2\2\2\u0162\u0163") - buf.write("\3\2\2\2\u0163\u0164\b \t\2\u0164\u0165\b \n\2\u0165D") - buf.write("\3\2\2\2\u0166\u0168\5?\36\2\u0167\u0166\3\2\2\2\u0167") - buf.write("\u0168\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u016a\7\177\2") - buf.write("\2\u016a\u016b\3\2\2\2\u016b\u016c\b!\6\2\u016cF\3\2\2") - buf.write("\2\u016d\u016e\7\60\2\2\u016eH\3\2\2\2\u016f\u0170\7]") - buf.write("\2\2\u0170\u0171\3\2\2\2\u0171\u0172\b#\13\2\u0172J\3") - buf.write("\2\2\2\u0173\u0174\7_\2\2\u0174\u0175\3\2\2\2\u0175\u0176") - buf.write("\b$\f\2\u0176L\3\2\2\2\u0177\u0178\5;\34\2\u0178\u0179") - buf.write("\3\2\2\2\u0179\u017a\b%\r\2\u017aN\3\2\2\2\u017b\u017d") - buf.write("\n\25\2\2\u017c\u017b\3\2\2\2\u017d\u017e\3\2\2\2\u017e") - buf.write("\u017c\3\2\2\2\u017e\u017f\3\2\2\2\u017fP\3\2\2\2\u0180") - buf.write("\u0181\5\33\f\2\u0181\u0182\3\2\2\2\u0182\u0183\b'\2") - buf.write("\2\u0183\u0184\b'\3\2\u0184R\3\2\2\2\u0185\u0186\7)\2") - buf.write("\2\u0186\u0187\3\2\2\2\u0187\u0188\b(\6\2\u0188T\3\2\2") - buf.write("\2\u0189\u018b\n\26\2\2\u018a\u0189\3\2\2\2\u018b\u018e") - buf.write("\3\2\2\2\u018c\u018a\3\2\2\2\u018c\u018d\3\2\2\2\u018d") - buf.write("\u018f\3\2\2\2\u018e\u018c\3\2\2\2\u018f\u0190\n\27\2") - buf.write("\2\u0190\u0191\3\2\2\2\u0191\u0192\b)\4\2\u0192V\3\2\2") - buf.write("\2\u0193\u0194\5\23\b\2\u0194\u0195\3\2\2\2\u0195\u0196") - buf.write("\b*\16\2\u0196X\3\2\2\2\u0197\u0199\5\r\5\2\u0198\u0197") - buf.write("\3\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2\2\2\u019a") - buf.write("\u019b\3\2\2\2\u019b\u019d\3\2\2\2\u019c\u019a\3\2\2\2") - buf.write("\u019d\u019e\7^\2\2\u019e\u019f\7)\2\2\u019f\u01a0\3\2") - buf.write("\2\2\u01a0\u01a1\b+\17\2\u01a1Z\3\2\2\2\u01a2\u01a4\5") - buf.write("\r\5\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3") - buf.write("\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\\\3\2\2\2\u01a7\u01a9") - buf.write("\7^\2\2\u01a8\u01a7\3\2\2\2\u01a9\u01aa\3\2\2\2\u01aa") - buf.write("\u01a8\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2\2\2") - buf.write("\u01ac\u01ad\b-\4\2\u01ad^\3\2\2\2\u01ae\u01af\7&\2\2") - buf.write("\u01af\u01b0\3\2\2\2\u01b0\u01b1\b.\4\2\u01b1`\3\2\2\2") - buf.write("\u01b2\u01b3\5\33\f\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5") - buf.write("\b/\2\2\u01b5\u01b6\b/\3\2\u01b6b\3\2\2\2\u01b7\u01b8") - buf.write("\7$\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01ba\b\60\20\2\u01ba") - buf.write("\u01bb\b\60\6\2\u01bbd\3\2\2\2\u01bc\u01be\n\30\2\2\u01bd") - buf.write("\u01bc\3\2\2\2\u01be\u01c1\3\2\2\2\u01bf\u01bd\3\2\2\2") - buf.write("\u01bf\u01c0\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1\u01bf\3") - buf.write("\2\2\2\u01c2\u01c3\n\31\2\2\u01c3\u01c4\3\2\2\2\u01c4") - buf.write("\u01c5\b\61\4\2\u01c5f\3\2\2\2\u01c6\u01c7\5\23\b\2\u01c7") - buf.write("\u01c8\3\2\2\2\u01c8\u01c9\b\62\16\2\u01c9h\3\2\2\2\u01ca") - buf.write("\u01cc\5\r\5\2\u01cb\u01ca\3\2\2\2\u01cc\u01cf\3\2\2\2") - buf.write("\u01cd\u01cb\3\2\2\2\u01cd\u01ce\3\2\2\2\u01ce\u01d0\3") - buf.write("\2\2\2\u01cf\u01cd\3\2\2\2\u01d0\u01d1\7^\2\2\u01d1\u01d2") - buf.write("\7$\2\2\u01d2\u01d3\3\2\2\2\u01d3\u01d4\b\63\17\2\u01d4") - buf.write("j\3\2\2\2\u01d5\u01d7\5\r\5\2\u01d6\u01d5\3\2\2\2\u01d7") - buf.write("\u01d8\3\2\2\2\u01d8\u01d6\3\2\2\2\u01d8\u01d9\3\2\2\2") - buf.write("\u01d9\u01da\3\2\2\2\u01da\u01db\b\64\21\2\u01dbl\3\2") - buf.write("\2\2\u01dc\u01de\7^\2\2\u01dd\u01dc\3\2\2\2\u01de\u01df") - buf.write("\3\2\2\2\u01df\u01dd\3\2\2\2\u01df\u01e0\3\2\2\2\u01e0") - buf.write("\u01e1\3\2\2\2\u01e1\u01e2\b\65\4\2\u01e2n\3\2\2\2\u01e3") - buf.write("\u01e4\7&\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e6\b\66\4\2") - buf.write("\u01e6p\3\2\2\2\66\2\3\4\5\6x}\u0080\u008d\u0095\u009f") - buf.write("\u00a4\u00b0\u00b6\u00bb\u00ca\u00ce\u00d2\u00d5\u00da") - buf.write("\u00de\u00e4\u00e9\u00ee\u00f1\u00f5\u00f9\u00fd\u0102") - buf.write("\u0106\u0110\u0113\u0120\u012b\u0130\u0132\u014c\u014e") - buf.write("\u0153\u0157\u015d\u0161\u0167\u017e\u018c\u019a\u01a5") - buf.write("\u01aa\u01bf\u01cd\u01d8\u01df\22\t\6\2\7\4\2\t\3\2\7") - buf.write("\3\2\6\2\2\7\5\2\7\6\2\t\16\2\4\3\2\t\f\2\t\r\2\t\24\2") - buf.write("\t\4\2\t\25\2\t\32\2\t\33\2") - return buf.getvalue() - - -class OmegaConfGrammarLexer(Lexer): - atn = ATNDeserializer().deserialize(serializedATN()) - - decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)] - - VALUE_MODE = 1 - INTERPOLATION_MODE = 2 - QUOTED_SINGLE_MODE = 3 - QUOTED_DOUBLE_MODE = 4 - - ANY_STR = 1 - ESC_INTER = 2 - TOP_ESC = 3 - INTER_OPEN = 4 - BRACE_OPEN = 5 - BRACE_CLOSE = 6 - QUOTE_OPEN_SINGLE = 7 - QUOTE_OPEN_DOUBLE = 8 - COMMA = 9 - BRACKET_OPEN = 10 - BRACKET_CLOSE = 11 - COLON = 12 - FLOAT = 13 - INT = 14 - BOOL = 15 - NULL = 16 - UNQUOTED_CHAR = 17 - ID = 18 - ESC = 19 - WS = 20 - INTER_CLOSE = 21 - DOT = 22 - INTER_KEY = 23 - MATCHING_QUOTE_CLOSE = 24 - QUOTED_ESC = 25 - DOLLAR = 26 - INTER_BRACKET_OPEN = 27 - INTER_BRACKET_CLOSE = 28 - - channelNames = ["DEFAULT_TOKEN_CHANNEL", "HIDDEN"] - - modeNames = [ - "DEFAULT_MODE", - "VALUE_MODE", - "INTERPOLATION_MODE", - "QUOTED_SINGLE_MODE", - "QUOTED_DOUBLE_MODE", - ] - - literalNames = ["", "'.'", "'['", "']'"] - - symbolicNames = [ - "", - "ANY_STR", - "ESC_INTER", - "TOP_ESC", - "INTER_OPEN", - "BRACE_OPEN", - "BRACE_CLOSE", - "QUOTE_OPEN_SINGLE", - "QUOTE_OPEN_DOUBLE", - "COMMA", - "BRACKET_OPEN", - "BRACKET_CLOSE", - "COLON", - "FLOAT", - "INT", - "BOOL", - "NULL", - "UNQUOTED_CHAR", - "ID", - "ESC", - "WS", - "INTER_CLOSE", - "DOT", - "INTER_KEY", - "MATCHING_QUOTE_CLOSE", - "QUOTED_ESC", - "DOLLAR", - "INTER_BRACKET_OPEN", - "INTER_BRACKET_CLOSE", - ] - - ruleNames = [ - "CHAR", - "DIGIT", - "INT_UNSIGNED", - "ESC_BACKSLASH", - "TOP_INTER_OPEN", - "ANY_STR", - "ESC_INTER", - "TOP_ESC", - "BACKSLASHES", - "DOLLAR", - "INTER_OPEN", - "BRACE_OPEN", - "BRACE_CLOSE", - "QUOTE_OPEN_SINGLE", - "QUOTE_OPEN_DOUBLE", - "COMMA", - "BRACKET_OPEN", - "BRACKET_CLOSE", - "COLON", - "POINT_FLOAT", - "EXPONENT_FLOAT", - "FLOAT", - "INT", - "BOOL", - "NULL", - "UNQUOTED_CHAR", - "ID", - "ESC", - "WS", - "NESTED_INTER_OPEN", - "INTER_COLON", - "INTER_CLOSE", - "DOT", - "INTER_BRACKET_OPEN", - "INTER_BRACKET_CLOSE", - "INTER_ID", - "INTER_KEY", - "QSINGLE_INTER_OPEN", - "MATCHING_QUOTE_CLOSE", - "QSINGLE_STR", - "QSINGLE_ESC_INTER", - "QSINGLE_ESC_QUOTE", - "QUOTED_ESC", - "QSINGLE_BACKSLASHES", - "QSINGLE_DOLLAR", - "QDOUBLE_INTER_OPEN", - "QDOUBLE_CLOSE", - "QDOUBLE_STR", - "QDOUBLE_ESC_INTER", - "QDOUBLE_ESC_QUOTE", - "QDOUBLE_ESC", - "QDOUBLE_BACKSLASHES", - "QDOUBLE_DOLLAR", - ] - - grammarFileName = "OmegaConfGrammarLexer.g4" - - def __init__(self, input=None, output: TextIO = sys.stdout): - super().__init__(input, output) - self.checkVersion("4.9.3") - self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) - self._actions = None - self._predicates = None diff --git a/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens b/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens deleted file mode 100644 index c22963b90..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarLexer.tokens +++ /dev/null @@ -1,31 +0,0 @@ -ANY_STR=1 -ESC_INTER=2 -TOP_ESC=3 -INTER_OPEN=4 -BRACE_OPEN=5 -BRACE_CLOSE=6 -QUOTE_OPEN_SINGLE=7 -QUOTE_OPEN_DOUBLE=8 -COMMA=9 -BRACKET_OPEN=10 -BRACKET_CLOSE=11 -COLON=12 -FLOAT=13 -INT=14 -BOOL=15 -NULL=16 -UNQUOTED_CHAR=17 -ID=18 -ESC=19 -WS=20 -INTER_CLOSE=21 -DOT=22 -INTER_KEY=23 -MATCHING_QUOTE_CLOSE=24 -QUOTED_ESC=25 -DOLLAR=26 -INTER_BRACKET_OPEN=27 -INTER_BRACKET_CLOSE=28 -'.'=22 -'['=27 -']'=28 diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.interp b/omegaconf/grammar/gen/OmegaConfGrammarParser.interp deleted file mode 100644 index b99c77c0a..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarParser.interp +++ /dev/null @@ -1,83 +0,0 @@ -token literal names: -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -null -'.' -null -null -null -null -'[' -']' - -token symbolic names: -null -ANY_STR -ESC_INTER -TOP_ESC -INTER_OPEN -BRACE_OPEN -BRACE_CLOSE -QUOTE_OPEN_SINGLE -QUOTE_OPEN_DOUBLE -COMMA -BRACKET_OPEN -BRACKET_CLOSE -COLON -FLOAT -INT -BOOL -NULL -UNQUOTED_CHAR -ID -ESC -WS -INTER_CLOSE -DOT -INTER_KEY -MATCHING_QUOTE_CLOSE -QUOTED_ESC -DOLLAR -INTER_BRACKET_OPEN -INTER_BRACKET_CLOSE - -rule names: -configValue -singleElement -text -element -listContainer -dictContainer -dictKeyValuePair -sequence -interpolation -interpolationNode -interpolationResolver -configKey -resolverName -quotedValue -primitive -dictKey - - -atn: -[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 3, 30, 183, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 6, 4, 47, 10, 4, 13, 4, 14, 4, 48, 3, 5, 3, 5, 3, 5, 3, 5, 5, 5, 55, 10, 5, 3, 6, 3, 6, 5, 6, 59, 10, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 67, 10, 7, 12, 7, 14, 7, 70, 11, 7, 5, 7, 72, 10, 7, 3, 7, 3, 7, 3, 8, 3, 8, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 5, 9, 83, 10, 9, 7, 9, 85, 10, 9, 12, 9, 14, 9, 88, 11, 9, 3, 9, 3, 9, 5, 9, 92, 10, 9, 6, 9, 94, 10, 9, 13, 9, 14, 9, 95, 5, 9, 98, 10, 9, 3, 10, 3, 10, 5, 10, 102, 10, 10, 3, 11, 3, 11, 7, 11, 106, 10, 11, 12, 11, 14, 11, 109, 11, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 5, 11, 116, 10, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 7, 11, 124, 10, 11, 12, 11, 14, 11, 127, 11, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 5, 12, 135, 10, 12, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, 5, 13, 142, 10, 13, 3, 14, 3, 14, 5, 14, 146, 10, 14, 3, 14, 3, 14, 3, 14, 5, 14, 151, 10, 14, 7, 14, 153, 10, 14, 12, 14, 14, 14, 156, 11, 14, 3, 15, 3, 15, 5, 15, 160, 10, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 6, 16, 174, 10, 16, 13, 16, 14, 16, 175, 3, 17, 6, 17, 179, 10, 17, 13, 17, 14, 17, 180, 3, 17, 2, 2, 18, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 2, 4, 3, 2, 9, 10, 3, 2, 15, 22, 2, 206, 2, 34, 3, 2, 2, 2, 4, 37, 3, 2, 2, 2, 6, 46, 3, 2, 2, 2, 8, 54, 3, 2, 2, 2, 10, 56, 3, 2, 2, 2, 12, 62, 3, 2, 2, 2, 14, 75, 3, 2, 2, 2, 16, 97, 3, 2, 2, 2, 18, 101, 3, 2, 2, 2, 20, 103, 3, 2, 2, 2, 22, 130, 3, 2, 2, 2, 24, 141, 3, 2, 2, 2, 26, 145, 3, 2, 2, 2, 28, 157, 3, 2, 2, 2, 30, 173, 3, 2, 2, 2, 32, 178, 3, 2, 2, 2, 34, 35, 5, 6, 4, 2, 35, 36, 7, 2, 2, 3, 36, 3, 3, 2, 2, 2, 37, 38, 5, 8, 5, 2, 38, 39, 7, 2, 2, 3, 39, 5, 3, 2, 2, 2, 40, 47, 5, 18, 10, 2, 41, 47, 7, 3, 2, 2, 42, 47, 7, 21, 2, 2, 43, 47, 7, 4, 2, 2, 44, 47, 7, 5, 2, 2, 45, 47, 7, 27, 2, 2, 46, 40, 3, 2, 2, 2, 46, 41, 3, 2, 2, 2, 46, 42, 3, 2, 2, 2, 46, 43, 3, 2, 2, 2, 46, 44, 3, 2, 2, 2, 46, 45, 3, 2, 2, 2, 47, 48, 3, 2, 2, 2, 48, 46, 3, 2, 2, 2, 48, 49, 3, 2, 2, 2, 49, 7, 3, 2, 2, 2, 50, 55, 5, 30, 16, 2, 51, 55, 5, 28, 15, 2, 52, 55, 5, 10, 6, 2, 53, 55, 5, 12, 7, 2, 54, 50, 3, 2, 2, 2, 54, 51, 3, 2, 2, 2, 54, 52, 3, 2, 2, 2, 54, 53, 3, 2, 2, 2, 55, 9, 3, 2, 2, 2, 56, 58, 7, 12, 2, 2, 57, 59, 5, 16, 9, 2, 58, 57, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 60, 3, 2, 2, 2, 60, 61, 7, 13, 2, 2, 61, 11, 3, 2, 2, 2, 62, 71, 7, 7, 2, 2, 63, 68, 5, 14, 8, 2, 64, 65, 7, 11, 2, 2, 65, 67, 5, 14, 8, 2, 66, 64, 3, 2, 2, 2, 67, 70, 3, 2, 2, 2, 68, 66, 3, 2, 2, 2, 68, 69, 3, 2, 2, 2, 69, 72, 3, 2, 2, 2, 70, 68, 3, 2, 2, 2, 71, 63, 3, 2, 2, 2, 71, 72, 3, 2, 2, 2, 72, 73, 3, 2, 2, 2, 73, 74, 7, 8, 2, 2, 74, 13, 3, 2, 2, 2, 75, 76, 5, 32, 17, 2, 76, 77, 7, 14, 2, 2, 77, 78, 5, 8, 5, 2, 78, 15, 3, 2, 2, 2, 79, 86, 5, 8, 5, 2, 80, 82, 7, 11, 2, 2, 81, 83, 5, 8, 5, 2, 82, 81, 3, 2, 2, 2, 82, 83, 3, 2, 2, 2, 83, 85, 3, 2, 2, 2, 84, 80, 3, 2, 2, 2, 85, 88, 3, 2, 2, 2, 86, 84, 3, 2, 2, 2, 86, 87, 3, 2, 2, 2, 87, 98, 3, 2, 2, 2, 88, 86, 3, 2, 2, 2, 89, 91, 7, 11, 2, 2, 90, 92, 5, 8, 5, 2, 91, 90, 3, 2, 2, 2, 91, 92, 3, 2, 2, 2, 92, 94, 3, 2, 2, 2, 93, 89, 3, 2, 2, 2, 94, 95, 3, 2, 2, 2, 95, 93, 3, 2, 2, 2, 95, 96, 3, 2, 2, 2, 96, 98, 3, 2, 2, 2, 97, 79, 3, 2, 2, 2, 97, 93, 3, 2, 2, 2, 98, 17, 3, 2, 2, 2, 99, 102, 5, 20, 11, 2, 100, 102, 5, 22, 12, 2, 101, 99, 3, 2, 2, 2, 101, 100, 3, 2, 2, 2, 102, 19, 3, 2, 2, 2, 103, 107, 7, 6, 2, 2, 104, 106, 7, 24, 2, 2, 105, 104, 3, 2, 2, 2, 106, 109, 3, 2, 2, 2, 107, 105, 3, 2, 2, 2, 107, 108, 3, 2, 2, 2, 108, 115, 3, 2, 2, 2, 109, 107, 3, 2, 2, 2, 110, 116, 5, 24, 13, 2, 111, 112, 7, 12, 2, 2, 112, 113, 5, 24, 13, 2, 113, 114, 7, 13, 2, 2, 114, 116, 3, 2, 2, 2, 115, 110, 3, 2, 2, 2, 115, 111, 3, 2, 2, 2, 116, 125, 3, 2, 2, 2, 117, 118, 7, 24, 2, 2, 118, 124, 5, 24, 13, 2, 119, 120, 7, 12, 2, 2, 120, 121, 5, 24, 13, 2, 121, 122, 7, 13, 2, 2, 122, 124, 3, 2, 2, 2, 123, 117, 3, 2, 2, 2, 123, 119, 3, 2, 2, 2, 124, 127, 3, 2, 2, 2, 125, 123, 3, 2, 2, 2, 125, 126, 3, 2, 2, 2, 126, 128, 3, 2, 2, 2, 127, 125, 3, 2, 2, 2, 128, 129, 7, 23, 2, 2, 129, 21, 3, 2, 2, 2, 130, 131, 7, 6, 2, 2, 131, 132, 5, 26, 14, 2, 132, 134, 7, 14, 2, 2, 133, 135, 5, 16, 9, 2, 134, 133, 3, 2, 2, 2, 134, 135, 3, 2, 2, 2, 135, 136, 3, 2, 2, 2, 136, 137, 7, 8, 2, 2, 137, 23, 3, 2, 2, 2, 138, 142, 5, 18, 10, 2, 139, 142, 7, 20, 2, 2, 140, 142, 7, 25, 2, 2, 141, 138, 3, 2, 2, 2, 141, 139, 3, 2, 2, 2, 141, 140, 3, 2, 2, 2, 142, 25, 3, 2, 2, 2, 143, 146, 5, 18, 10, 2, 144, 146, 7, 20, 2, 2, 145, 143, 3, 2, 2, 2, 145, 144, 3, 2, 2, 2, 146, 154, 3, 2, 2, 2, 147, 150, 7, 24, 2, 2, 148, 151, 5, 18, 10, 2, 149, 151, 7, 20, 2, 2, 150, 148, 3, 2, 2, 2, 150, 149, 3, 2, 2, 2, 151, 153, 3, 2, 2, 2, 152, 147, 3, 2, 2, 2, 153, 156, 3, 2, 2, 2, 154, 152, 3, 2, 2, 2, 154, 155, 3, 2, 2, 2, 155, 27, 3, 2, 2, 2, 156, 154, 3, 2, 2, 2, 157, 159, 9, 2, 2, 2, 158, 160, 5, 6, 4, 2, 159, 158, 3, 2, 2, 2, 159, 160, 3, 2, 2, 2, 160, 161, 3, 2, 2, 2, 161, 162, 7, 26, 2, 2, 162, 29, 3, 2, 2, 2, 163, 174, 7, 20, 2, 2, 164, 174, 7, 18, 2, 2, 165, 174, 7, 16, 2, 2, 166, 174, 7, 15, 2, 2, 167, 174, 7, 17, 2, 2, 168, 174, 7, 19, 2, 2, 169, 174, 7, 14, 2, 2, 170, 174, 7, 21, 2, 2, 171, 174, 7, 22, 2, 2, 172, 174, 5, 18, 10, 2, 173, 163, 3, 2, 2, 2, 173, 164, 3, 2, 2, 2, 173, 165, 3, 2, 2, 2, 173, 166, 3, 2, 2, 2, 173, 167, 3, 2, 2, 2, 173, 168, 3, 2, 2, 2, 173, 169, 3, 2, 2, 2, 173, 170, 3, 2, 2, 2, 173, 171, 3, 2, 2, 2, 173, 172, 3, 2, 2, 2, 174, 175, 3, 2, 2, 2, 175, 173, 3, 2, 2, 2, 175, 176, 3, 2, 2, 2, 176, 31, 3, 2, 2, 2, 177, 179, 9, 3, 2, 2, 178, 177, 3, 2, 2, 2, 179, 180, 3, 2, 2, 2, 180, 178, 3, 2, 2, 2, 180, 181, 3, 2, 2, 2, 181, 33, 3, 2, 2, 2, 27, 46, 48, 54, 58, 68, 71, 82, 86, 91, 95, 97, 101, 107, 115, 123, 125, 134, 141, 145, 150, 154, 159, 173, 175, 180] \ No newline at end of file diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.py b/omegaconf/grammar/gen/OmegaConfGrammarParser.py deleted file mode 100644 index 81cde7563..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarParser.py +++ /dev/null @@ -1,1745 +0,0 @@ -# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 -# encoding: utf-8 -from antlr4 import * -from io import StringIO -import sys - -if sys.version_info[1] > 5: - from typing import TextIO -else: - from typing.io import TextIO - - -def serializedATN(): - with StringIO() as buf: - buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3\36") - buf.write("\u00b7\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7") - buf.write("\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r\4\16") - buf.write("\t\16\4\17\t\17\4\20\t\20\4\21\t\21\3\2\3\2\3\2\3\3\3") - buf.write("\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\6\4/\n\4\r\4\16\4\60\3") - buf.write("\5\3\5\3\5\3\5\5\5\67\n\5\3\6\3\6\5\6;\n\6\3\6\3\6\3\7") - buf.write("\3\7\3\7\3\7\7\7C\n\7\f\7\16\7F\13\7\5\7H\n\7\3\7\3\7") - buf.write("\3\b\3\b\3\b\3\b\3\t\3\t\3\t\5\tS\n\t\7\tU\n\t\f\t\16") - buf.write("\tX\13\t\3\t\3\t\5\t\\\n\t\6\t^\n\t\r\t\16\t_\5\tb\n\t") - buf.write("\3\n\3\n\5\nf\n\n\3\13\3\13\7\13j\n\13\f\13\16\13m\13") - buf.write("\13\3\13\3\13\3\13\3\13\3\13\5\13t\n\13\3\13\3\13\3\13") - buf.write("\3\13\3\13\3\13\7\13|\n\13\f\13\16\13\177\13\13\3\13\3") - buf.write("\13\3\f\3\f\3\f\3\f\5\f\u0087\n\f\3\f\3\f\3\r\3\r\3\r") - buf.write("\5\r\u008e\n\r\3\16\3\16\5\16\u0092\n\16\3\16\3\16\3\16") - buf.write("\5\16\u0097\n\16\7\16\u0099\n\16\f\16\16\16\u009c\13\16") - buf.write("\3\17\3\17\5\17\u00a0\n\17\3\17\3\17\3\20\3\20\3\20\3") - buf.write("\20\3\20\3\20\3\20\3\20\3\20\3\20\6\20\u00ae\n\20\r\20") - buf.write("\16\20\u00af\3\21\6\21\u00b3\n\21\r\21\16\21\u00b4\3\21") - buf.write("\2\2\22\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \2\4\3") - buf.write('\2\t\n\3\2\17\26\2\u00ce\2"\3\2\2\2\4%\3\2\2\2\6.\3\2') - buf.write("\2\2\b\66\3\2\2\2\n8\3\2\2\2\f>\3\2\2\2\16K\3\2\2\2\20") - buf.write("a\3\2\2\2\22e\3\2\2\2\24g\3\2\2\2\26\u0082\3\2\2\2\30") - buf.write("\u008d\3\2\2\2\32\u0091\3\2\2\2\34\u009d\3\2\2\2\36\u00ad") - buf.write('\3\2\2\2 \u00b2\3\2\2\2"#\5\6\4\2#$\7\2\2\3$\3\3\2\2') - buf.write("\2%&\5\b\5\2&'\7\2\2\3'\5\3\2\2\2(/\5\22\n\2)/\7\3\2") - buf.write("\2*/\7\25\2\2+/\7\4\2\2,/\7\5\2\2-/\7\33\2\2.(\3\2\2\2") - buf.write(".)\3\2\2\2.*\3\2\2\2.+\3\2\2\2.,\3\2\2\2.-\3\2\2\2/\60") - buf.write("\3\2\2\2\60.\3\2\2\2\60\61\3\2\2\2\61\7\3\2\2\2\62\67") - buf.write("\5\36\20\2\63\67\5\34\17\2\64\67\5\n\6\2\65\67\5\f\7\2") - buf.write("\66\62\3\2\2\2\66\63\3\2\2\2\66\64\3\2\2\2\66\65\3\2\2") - buf.write("\2\67\t\3\2\2\28:\7\f\2\29;\5\20\t\2:9\3\2\2\2:;\3\2\2") - buf.write("\2;<\3\2\2\2<=\7\r\2\2=\13\3\2\2\2>G\7\7\2\2?D\5\16\b") - buf.write("\2@A\7\13\2\2AC\5\16\b\2B@\3\2\2\2CF\3\2\2\2DB\3\2\2\2") - buf.write("DE\3\2\2\2EH\3\2\2\2FD\3\2\2\2G?\3\2\2\2GH\3\2\2\2HI\3") - buf.write("\2\2\2IJ\7\b\2\2J\r\3\2\2\2KL\5 \21\2LM\7\16\2\2MN\5\b") - buf.write("\5\2N\17\3\2\2\2OV\5\b\5\2PR\7\13\2\2QS\5\b\5\2RQ\3\2") - buf.write("\2\2RS\3\2\2\2SU\3\2\2\2TP\3\2\2\2UX\3\2\2\2VT\3\2\2\2") - buf.write("VW\3\2\2\2Wb\3\2\2\2XV\3\2\2\2Y[\7\13\2\2Z\\\5\b\5\2[") - buf.write("Z\3\2\2\2[\\\3\2\2\2\\^\3\2\2\2]Y\3\2\2\2^_\3\2\2\2_]") - buf.write("\3\2\2\2_`\3\2\2\2`b\3\2\2\2aO\3\2\2\2a]\3\2\2\2b\21\3") - buf.write("\2\2\2cf\5\24\13\2df\5\26\f\2ec\3\2\2\2ed\3\2\2\2f\23") - buf.write("\3\2\2\2gk\7\6\2\2hj\7\30\2\2ih\3\2\2\2jm\3\2\2\2ki\3") - buf.write("\2\2\2kl\3\2\2\2ls\3\2\2\2mk\3\2\2\2nt\5\30\r\2op\7\f") - buf.write("\2\2pq\5\30\r\2qr\7\r\2\2rt\3\2\2\2sn\3\2\2\2so\3\2\2") - buf.write("\2t}\3\2\2\2uv\7\30\2\2v|\5\30\r\2wx\7\f\2\2xy\5\30\r") - buf.write("\2yz\7\r\2\2z|\3\2\2\2{u\3\2\2\2{w\3\2\2\2|\177\3\2\2") - buf.write("\2}{\3\2\2\2}~\3\2\2\2~\u0080\3\2\2\2\177}\3\2\2\2\u0080") - buf.write("\u0081\7\27\2\2\u0081\25\3\2\2\2\u0082\u0083\7\6\2\2\u0083") - buf.write("\u0084\5\32\16\2\u0084\u0086\7\16\2\2\u0085\u0087\5\20") - buf.write("\t\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088") - buf.write("\3\2\2\2\u0088\u0089\7\b\2\2\u0089\27\3\2\2\2\u008a\u008e") - buf.write("\5\22\n\2\u008b\u008e\7\24\2\2\u008c\u008e\7\31\2\2\u008d") - buf.write("\u008a\3\2\2\2\u008d\u008b\3\2\2\2\u008d\u008c\3\2\2\2") - buf.write("\u008e\31\3\2\2\2\u008f\u0092\5\22\n\2\u0090\u0092\7\24") - buf.write("\2\2\u0091\u008f\3\2\2\2\u0091\u0090\3\2\2\2\u0092\u009a") - buf.write("\3\2\2\2\u0093\u0096\7\30\2\2\u0094\u0097\5\22\n\2\u0095") - buf.write("\u0097\7\24\2\2\u0096\u0094\3\2\2\2\u0096\u0095\3\2\2") - buf.write("\2\u0097\u0099\3\2\2\2\u0098\u0093\3\2\2\2\u0099\u009c") - buf.write("\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2\2\2\u009b") - buf.write("\33\3\2\2\2\u009c\u009a\3\2\2\2\u009d\u009f\t\2\2\2\u009e") - buf.write("\u00a0\5\6\4\2\u009f\u009e\3\2\2\2\u009f\u00a0\3\2\2\2") - buf.write("\u00a0\u00a1\3\2\2\2\u00a1\u00a2\7\32\2\2\u00a2\35\3\2") - buf.write("\2\2\u00a3\u00ae\7\24\2\2\u00a4\u00ae\7\22\2\2\u00a5\u00ae") - buf.write("\7\20\2\2\u00a6\u00ae\7\17\2\2\u00a7\u00ae\7\21\2\2\u00a8") - buf.write("\u00ae\7\23\2\2\u00a9\u00ae\7\16\2\2\u00aa\u00ae\7\25") - buf.write("\2\2\u00ab\u00ae\7\26\2\2\u00ac\u00ae\5\22\n\2\u00ad\u00a3") - buf.write("\3\2\2\2\u00ad\u00a4\3\2\2\2\u00ad\u00a5\3\2\2\2\u00ad") - buf.write("\u00a6\3\2\2\2\u00ad\u00a7\3\2\2\2\u00ad\u00a8\3\2\2\2") - buf.write("\u00ad\u00a9\3\2\2\2\u00ad\u00aa\3\2\2\2\u00ad\u00ab\3") - buf.write("\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00ad") - buf.write("\3\2\2\2\u00af\u00b0\3\2\2\2\u00b0\37\3\2\2\2\u00b1\u00b3") - buf.write("\t\3\2\2\u00b2\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4") - buf.write("\u00b2\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5!\3\2\2\2\33.") - buf.write("\60\66:DGRV[_aeks{}\u0086\u008d\u0091\u0096\u009a\u009f") - buf.write("\u00ad\u00af\u00b4") - return buf.getvalue() - - -class OmegaConfGrammarParser(Parser): - grammarFileName = "OmegaConfGrammarParser.g4" - - atn = ATNDeserializer().deserialize(serializedATN()) - - decisionsToDFA = [DFA(ds, i) for i, ds in enumerate(atn.decisionToState)] - - sharedContextCache = PredictionContextCache() - - literalNames = [ - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "'.'", - "", - "", - "", - "", - "'['", - "']'", - ] - - symbolicNames = [ - "", - "ANY_STR", - "ESC_INTER", - "TOP_ESC", - "INTER_OPEN", - "BRACE_OPEN", - "BRACE_CLOSE", - "QUOTE_OPEN_SINGLE", - "QUOTE_OPEN_DOUBLE", - "COMMA", - "BRACKET_OPEN", - "BRACKET_CLOSE", - "COLON", - "FLOAT", - "INT", - "BOOL", - "NULL", - "UNQUOTED_CHAR", - "ID", - "ESC", - "WS", - "INTER_CLOSE", - "DOT", - "INTER_KEY", - "MATCHING_QUOTE_CLOSE", - "QUOTED_ESC", - "DOLLAR", - "INTER_BRACKET_OPEN", - "INTER_BRACKET_CLOSE", - ] - - RULE_configValue = 0 - RULE_singleElement = 1 - RULE_text = 2 - RULE_element = 3 - RULE_listContainer = 4 - RULE_dictContainer = 5 - RULE_dictKeyValuePair = 6 - RULE_sequence = 7 - RULE_interpolation = 8 - RULE_interpolationNode = 9 - RULE_interpolationResolver = 10 - RULE_configKey = 11 - RULE_resolverName = 12 - RULE_quotedValue = 13 - RULE_primitive = 14 - RULE_dictKey = 15 - - ruleNames = [ - "configValue", - "singleElement", - "text", - "element", - "listContainer", - "dictContainer", - "dictKeyValuePair", - "sequence", - "interpolation", - "interpolationNode", - "interpolationResolver", - "configKey", - "resolverName", - "quotedValue", - "primitive", - "dictKey", - ] - - EOF = Token.EOF - ANY_STR = 1 - ESC_INTER = 2 - TOP_ESC = 3 - INTER_OPEN = 4 - BRACE_OPEN = 5 - BRACE_CLOSE = 6 - QUOTE_OPEN_SINGLE = 7 - QUOTE_OPEN_DOUBLE = 8 - COMMA = 9 - BRACKET_OPEN = 10 - BRACKET_CLOSE = 11 - COLON = 12 - FLOAT = 13 - INT = 14 - BOOL = 15 - NULL = 16 - UNQUOTED_CHAR = 17 - ID = 18 - ESC = 19 - WS = 20 - INTER_CLOSE = 21 - DOT = 22 - INTER_KEY = 23 - MATCHING_QUOTE_CLOSE = 24 - QUOTED_ESC = 25 - DOLLAR = 26 - INTER_BRACKET_OPEN = 27 - INTER_BRACKET_CLOSE = 28 - - def __init__(self, input: TokenStream, output: TextIO = sys.stdout): - super().__init__(input, output) - self.checkVersion("4.9.3") - self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) - self._predicates = None - - class ConfigValueContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def text(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.TextContext, 0) - - def EOF(self): - return self.getToken(OmegaConfGrammarParser.EOF, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_configValue - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterConfigValue"): - listener.enterConfigValue(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitConfigValue"): - listener.exitConfigValue(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitConfigValue"): - return visitor.visitConfigValue(self) - else: - return visitor.visitChildren(self) - - def configValue(self): - localctx = OmegaConfGrammarParser.ConfigValueContext(self, self._ctx, self.state) - self.enterRule(localctx, 0, self.RULE_configValue) - try: - self.enterOuterAlt(localctx, 1) - self.state = 32 - self.text() - self.state = 33 - self.match(OmegaConfGrammarParser.EOF) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class SingleElementContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def element(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, 0) - - def EOF(self): - return self.getToken(OmegaConfGrammarParser.EOF, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_singleElement - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterSingleElement"): - listener.enterSingleElement(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitSingleElement"): - listener.exitSingleElement(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitSingleElement"): - return visitor.visitSingleElement(self) - else: - return visitor.visitChildren(self) - - def singleElement(self): - localctx = OmegaConfGrammarParser.SingleElementContext(self, self._ctx, self.state) - self.enterRule(localctx, 2, self.RULE_singleElement) - try: - self.enterOuterAlt(localctx, 1) - self.state = 35 - self.element() - self.state = 36 - self.match(OmegaConfGrammarParser.EOF) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class TextContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def interpolation(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) - - def ANY_STR(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ANY_STR) - else: - return self.getToken(OmegaConfGrammarParser.ANY_STR, i) - - def ESC(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ESC) - else: - return self.getToken(OmegaConfGrammarParser.ESC, i) - - def ESC_INTER(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ESC_INTER) - else: - return self.getToken(OmegaConfGrammarParser.ESC_INTER, i) - - def TOP_ESC(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.TOP_ESC) - else: - return self.getToken(OmegaConfGrammarParser.TOP_ESC, i) - - def QUOTED_ESC(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.QUOTED_ESC) - else: - return self.getToken(OmegaConfGrammarParser.QUOTED_ESC, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_text - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterText"): - listener.enterText(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitText"): - listener.exitText(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitText"): - return visitor.visitText(self) - else: - return visitor.visitChildren(self) - - def text(self): - localctx = OmegaConfGrammarParser.TextContext(self, self._ctx, self.state) - self.enterRule(localctx, 4, self.RULE_text) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 44 - self._errHandler.sync(self) - _la = self._input.LA(1) - while True: - self.state = 44 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.INTER_OPEN]: - self.state = 38 - self.interpolation() - pass - elif token in [OmegaConfGrammarParser.ANY_STR]: - self.state = 39 - self.match(OmegaConfGrammarParser.ANY_STR) - pass - elif token in [OmegaConfGrammarParser.ESC]: - self.state = 40 - self.match(OmegaConfGrammarParser.ESC) - pass - elif token in [OmegaConfGrammarParser.ESC_INTER]: - self.state = 41 - self.match(OmegaConfGrammarParser.ESC_INTER) - pass - elif token in [OmegaConfGrammarParser.TOP_ESC]: - self.state = 42 - self.match(OmegaConfGrammarParser.TOP_ESC) - pass - elif token in [OmegaConfGrammarParser.QUOTED_ESC]: - self.state = 43 - self.match(OmegaConfGrammarParser.QUOTED_ESC) - pass - else: - raise NoViableAltException(self) - - self.state = 46 - self._errHandler.sync(self) - _la = self._input.LA(1) - if not ( - ( - ((_la) & ~0x3F) == 0 - and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.ANY_STR) - | (1 << OmegaConfGrammarParser.ESC_INTER) - | (1 << OmegaConfGrammarParser.TOP_ESC) - | (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.QUOTED_ESC) - ) - ) - != 0 - ) - ): - break - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class ElementContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def primitive(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.PrimitiveContext, 0) - - def quotedValue(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.QuotedValueContext, 0) - - def listContainer(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.ListContainerContext, 0) - - def dictContainer(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.DictContainerContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_element - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterElement"): - listener.enterElement(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitElement"): - listener.exitElement(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitElement"): - return visitor.visitElement(self) - else: - return visitor.visitChildren(self) - - def element(self): - localctx = OmegaConfGrammarParser.ElementContext(self, self._ctx, self.state) - self.enterRule(localctx, 6, self.RULE_element) - try: - self.state = 52 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [ - OmegaConfGrammarParser.INTER_OPEN, - OmegaConfGrammarParser.COLON, - OmegaConfGrammarParser.FLOAT, - OmegaConfGrammarParser.INT, - OmegaConfGrammarParser.BOOL, - OmegaConfGrammarParser.NULL, - OmegaConfGrammarParser.UNQUOTED_CHAR, - OmegaConfGrammarParser.ID, - OmegaConfGrammarParser.ESC, - OmegaConfGrammarParser.WS, - ]: - self.enterOuterAlt(localctx, 1) - self.state = 48 - self.primitive() - pass - elif token in [ - OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, - OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, - ]: - self.enterOuterAlt(localctx, 2) - self.state = 49 - self.quotedValue() - pass - elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: - self.enterOuterAlt(localctx, 3) - self.state = 50 - self.listContainer() - pass - elif token in [OmegaConfGrammarParser.BRACE_OPEN]: - self.enterOuterAlt(localctx, 4) - self.state = 51 - self.dictContainer() - pass - else: - raise NoViableAltException(self) - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class ListContainerContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def BRACKET_OPEN(self): - return self.getToken(OmegaConfGrammarParser.BRACKET_OPEN, 0) - - def BRACKET_CLOSE(self): - return self.getToken(OmegaConfGrammarParser.BRACKET_CLOSE, 0) - - def sequence(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.SequenceContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_listContainer - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterListContainer"): - listener.enterListContainer(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitListContainer"): - listener.exitListContainer(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitListContainer"): - return visitor.visitListContainer(self) - else: - return visitor.visitChildren(self) - - def listContainer(self): - localctx = OmegaConfGrammarParser.ListContainerContext(self, self._ctx, self.state) - self.enterRule(localctx, 8, self.RULE_listContainer) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 54 - self.match(OmegaConfGrammarParser.BRACKET_OPEN) - self.state = 56 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.BRACE_OPEN) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) - | (1 << OmegaConfGrammarParser.COMMA) - | (1 << OmegaConfGrammarParser.BRACKET_OPEN) - | (1 << OmegaConfGrammarParser.COLON) - | (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) != 0: - self.state = 55 - self.sequence() - - self.state = 58 - self.match(OmegaConfGrammarParser.BRACKET_CLOSE) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class DictContainerContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def BRACE_OPEN(self): - return self.getToken(OmegaConfGrammarParser.BRACE_OPEN, 0) - - def BRACE_CLOSE(self): - return self.getToken(OmegaConfGrammarParser.BRACE_CLOSE, 0) - - def dictKeyValuePair(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.DictKeyValuePairContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.DictKeyValuePairContext, i) - - def COMMA(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.COMMA) - else: - return self.getToken(OmegaConfGrammarParser.COMMA, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_dictContainer - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterDictContainer"): - listener.enterDictContainer(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitDictContainer"): - listener.exitDictContainer(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitDictContainer"): - return visitor.visitDictContainer(self) - else: - return visitor.visitChildren(self) - - def dictContainer(self): - localctx = OmegaConfGrammarParser.DictContainerContext(self, self._ctx, self.state) - self.enterRule(localctx, 10, self.RULE_dictContainer) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 60 - self.match(OmegaConfGrammarParser.BRACE_OPEN) - self.state = 69 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) != 0: - self.state = 61 - self.dictKeyValuePair() - self.state = 66 - self._errHandler.sync(self) - _la = self._input.LA(1) - while _la == OmegaConfGrammarParser.COMMA: - self.state = 62 - self.match(OmegaConfGrammarParser.COMMA) - self.state = 63 - self.dictKeyValuePair() - self.state = 68 - self._errHandler.sync(self) - _la = self._input.LA(1) - - self.state = 71 - self.match(OmegaConfGrammarParser.BRACE_CLOSE) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class DictKeyValuePairContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def dictKey(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.DictKeyContext, 0) - - def COLON(self): - return self.getToken(OmegaConfGrammarParser.COLON, 0) - - def element(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_dictKeyValuePair - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterDictKeyValuePair"): - listener.enterDictKeyValuePair(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitDictKeyValuePair"): - listener.exitDictKeyValuePair(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitDictKeyValuePair"): - return visitor.visitDictKeyValuePair(self) - else: - return visitor.visitChildren(self) - - def dictKeyValuePair(self): - localctx = OmegaConfGrammarParser.DictKeyValuePairContext(self, self._ctx, self.state) - self.enterRule(localctx, 12, self.RULE_dictKeyValuePair) - try: - self.enterOuterAlt(localctx, 1) - self.state = 73 - self.dictKey() - self.state = 74 - self.match(OmegaConfGrammarParser.COLON) - self.state = 75 - self.element() - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class SequenceContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def element(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.ElementContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.ElementContext, i) - - def COMMA(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.COMMA) - else: - return self.getToken(OmegaConfGrammarParser.COMMA, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_sequence - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterSequence"): - listener.enterSequence(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitSequence"): - listener.exitSequence(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitSequence"): - return visitor.visitSequence(self) - else: - return visitor.visitChildren(self) - - def sequence(self): - localctx = OmegaConfGrammarParser.SequenceContext(self, self._ctx, self.state) - self.enterRule(localctx, 14, self.RULE_sequence) - self._la = 0 # Token type - try: - self.state = 95 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [ - OmegaConfGrammarParser.INTER_OPEN, - OmegaConfGrammarParser.BRACE_OPEN, - OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, - OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, - OmegaConfGrammarParser.BRACKET_OPEN, - OmegaConfGrammarParser.COLON, - OmegaConfGrammarParser.FLOAT, - OmegaConfGrammarParser.INT, - OmegaConfGrammarParser.BOOL, - OmegaConfGrammarParser.NULL, - OmegaConfGrammarParser.UNQUOTED_CHAR, - OmegaConfGrammarParser.ID, - OmegaConfGrammarParser.ESC, - OmegaConfGrammarParser.WS, - ]: - self.enterOuterAlt(localctx, 1) - self.state = 77 - self.element() - self.state = 84 - self._errHandler.sync(self) - _la = self._input.LA(1) - while _la == OmegaConfGrammarParser.COMMA: - self.state = 78 - self.match(OmegaConfGrammarParser.COMMA) - self.state = 80 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.BRACE_OPEN) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) - | (1 << OmegaConfGrammarParser.BRACKET_OPEN) - | (1 << OmegaConfGrammarParser.COLON) - | (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) != 0: - self.state = 79 - self.element() - - self.state = 86 - self._errHandler.sync(self) - _la = self._input.LA(1) - - pass - elif token in [OmegaConfGrammarParser.COMMA]: - self.enterOuterAlt(localctx, 2) - self.state = 91 - self._errHandler.sync(self) - _la = self._input.LA(1) - while True: - self.state = 87 - self.match(OmegaConfGrammarParser.COMMA) - self.state = 89 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.BRACE_OPEN) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) - | (1 << OmegaConfGrammarParser.BRACKET_OPEN) - | (1 << OmegaConfGrammarParser.COLON) - | (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) != 0: - self.state = 88 - self.element() - - self.state = 93 - self._errHandler.sync(self) - _la = self._input.LA(1) - if not (_la == OmegaConfGrammarParser.COMMA): - break - - pass - else: - raise NoViableAltException(self) - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class InterpolationContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def interpolationNode(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationNodeContext, 0) - - def interpolationResolver(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationResolverContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_interpolation - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterInterpolation"): - listener.enterInterpolation(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitInterpolation"): - listener.exitInterpolation(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitInterpolation"): - return visitor.visitInterpolation(self) - else: - return visitor.visitChildren(self) - - def interpolation(self): - localctx = OmegaConfGrammarParser.InterpolationContext(self, self._ctx, self.state) - self.enterRule(localctx, 16, self.RULE_interpolation) - try: - self.state = 99 - self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input, 11, self._ctx) - if la_ == 1: - self.enterOuterAlt(localctx, 1) - self.state = 97 - self.interpolationNode() - pass - - elif la_ == 2: - self.enterOuterAlt(localctx, 2) - self.state = 98 - self.interpolationResolver() - pass - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class InterpolationNodeContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def INTER_OPEN(self): - return self.getToken(OmegaConfGrammarParser.INTER_OPEN, 0) - - def INTER_CLOSE(self): - return self.getToken(OmegaConfGrammarParser.INTER_CLOSE, 0) - - def configKey(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.ConfigKeyContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.ConfigKeyContext, i) - - def BRACKET_OPEN(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.BRACKET_OPEN) - else: - return self.getToken(OmegaConfGrammarParser.BRACKET_OPEN, i) - - def BRACKET_CLOSE(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.BRACKET_CLOSE) - else: - return self.getToken(OmegaConfGrammarParser.BRACKET_CLOSE, i) - - def DOT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.DOT) - else: - return self.getToken(OmegaConfGrammarParser.DOT, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_interpolationNode - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterInterpolationNode"): - listener.enterInterpolationNode(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitInterpolationNode"): - listener.exitInterpolationNode(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitInterpolationNode"): - return visitor.visitInterpolationNode(self) - else: - return visitor.visitChildren(self) - - def interpolationNode(self): - localctx = OmegaConfGrammarParser.InterpolationNodeContext(self, self._ctx, self.state) - self.enterRule(localctx, 18, self.RULE_interpolationNode) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 101 - self.match(OmegaConfGrammarParser.INTER_OPEN) - self.state = 105 - self._errHandler.sync(self) - _la = self._input.LA(1) - while _la == OmegaConfGrammarParser.DOT: - self.state = 102 - self.match(OmegaConfGrammarParser.DOT) - self.state = 107 - self._errHandler.sync(self) - _la = self._input.LA(1) - - self.state = 113 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [ - OmegaConfGrammarParser.INTER_OPEN, - OmegaConfGrammarParser.ID, - OmegaConfGrammarParser.INTER_KEY, - ]: - self.state = 108 - self.configKey() - pass - elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: - self.state = 109 - self.match(OmegaConfGrammarParser.BRACKET_OPEN) - self.state = 110 - self.configKey() - self.state = 111 - self.match(OmegaConfGrammarParser.BRACKET_CLOSE) - pass - else: - raise NoViableAltException(self) - - self.state = 123 - self._errHandler.sync(self) - _la = self._input.LA(1) - while _la == OmegaConfGrammarParser.BRACKET_OPEN or _la == OmegaConfGrammarParser.DOT: - self.state = 121 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.DOT]: - self.state = 115 - self.match(OmegaConfGrammarParser.DOT) - self.state = 116 - self.configKey() - pass - elif token in [OmegaConfGrammarParser.BRACKET_OPEN]: - self.state = 117 - self.match(OmegaConfGrammarParser.BRACKET_OPEN) - self.state = 118 - self.configKey() - self.state = 119 - self.match(OmegaConfGrammarParser.BRACKET_CLOSE) - pass - else: - raise NoViableAltException(self) - - self.state = 125 - self._errHandler.sync(self) - _la = self._input.LA(1) - - self.state = 126 - self.match(OmegaConfGrammarParser.INTER_CLOSE) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class InterpolationResolverContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def INTER_OPEN(self): - return self.getToken(OmegaConfGrammarParser.INTER_OPEN, 0) - - def resolverName(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.ResolverNameContext, 0) - - def COLON(self): - return self.getToken(OmegaConfGrammarParser.COLON, 0) - - def BRACE_CLOSE(self): - return self.getToken(OmegaConfGrammarParser.BRACE_CLOSE, 0) - - def sequence(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.SequenceContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_interpolationResolver - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterInterpolationResolver"): - listener.enterInterpolationResolver(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitInterpolationResolver"): - listener.exitInterpolationResolver(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitInterpolationResolver"): - return visitor.visitInterpolationResolver(self) - else: - return visitor.visitChildren(self) - - def interpolationResolver(self): - localctx = OmegaConfGrammarParser.InterpolationResolverContext(self, self._ctx, self.state) - self.enterRule(localctx, 20, self.RULE_interpolationResolver) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 128 - self.match(OmegaConfGrammarParser.INTER_OPEN) - self.state = 129 - self.resolverName() - self.state = 130 - self.match(OmegaConfGrammarParser.COLON) - self.state = 132 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.BRACE_OPEN) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_SINGLE) - | (1 << OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE) - | (1 << OmegaConfGrammarParser.COMMA) - | (1 << OmegaConfGrammarParser.BRACKET_OPEN) - | (1 << OmegaConfGrammarParser.COLON) - | (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) != 0: - self.state = 131 - self.sequence() - - self.state = 134 - self.match(OmegaConfGrammarParser.BRACE_CLOSE) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class ConfigKeyContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def interpolation(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, 0) - - def ID(self): - return self.getToken(OmegaConfGrammarParser.ID, 0) - - def INTER_KEY(self): - return self.getToken(OmegaConfGrammarParser.INTER_KEY, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_configKey - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterConfigKey"): - listener.enterConfigKey(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitConfigKey"): - listener.exitConfigKey(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitConfigKey"): - return visitor.visitConfigKey(self) - else: - return visitor.visitChildren(self) - - def configKey(self): - localctx = OmegaConfGrammarParser.ConfigKeyContext(self, self._ctx, self.state) - self.enterRule(localctx, 22, self.RULE_configKey) - try: - self.state = 139 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.INTER_OPEN]: - self.enterOuterAlt(localctx, 1) - self.state = 136 - self.interpolation() - pass - elif token in [OmegaConfGrammarParser.ID]: - self.enterOuterAlt(localctx, 2) - self.state = 137 - self.match(OmegaConfGrammarParser.ID) - pass - elif token in [OmegaConfGrammarParser.INTER_KEY]: - self.enterOuterAlt(localctx, 3) - self.state = 138 - self.match(OmegaConfGrammarParser.INTER_KEY) - pass - else: - raise NoViableAltException(self) - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class ResolverNameContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def interpolation(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) - - def ID(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ID) - else: - return self.getToken(OmegaConfGrammarParser.ID, i) - - def DOT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.DOT) - else: - return self.getToken(OmegaConfGrammarParser.DOT, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_resolverName - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterResolverName"): - listener.enterResolverName(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitResolverName"): - listener.exitResolverName(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitResolverName"): - return visitor.visitResolverName(self) - else: - return visitor.visitChildren(self) - - def resolverName(self): - localctx = OmegaConfGrammarParser.ResolverNameContext(self, self._ctx, self.state) - self.enterRule(localctx, 24, self.RULE_resolverName) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 143 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.INTER_OPEN]: - self.state = 141 - self.interpolation() - pass - elif token in [OmegaConfGrammarParser.ID]: - self.state = 142 - self.match(OmegaConfGrammarParser.ID) - pass - else: - raise NoViableAltException(self) - - self.state = 152 - self._errHandler.sync(self) - _la = self._input.LA(1) - while _la == OmegaConfGrammarParser.DOT: - self.state = 145 - self.match(OmegaConfGrammarParser.DOT) - self.state = 148 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.INTER_OPEN]: - self.state = 146 - self.interpolation() - pass - elif token in [OmegaConfGrammarParser.ID]: - self.state = 147 - self.match(OmegaConfGrammarParser.ID) - pass - else: - raise NoViableAltException(self) - - self.state = 154 - self._errHandler.sync(self) - _la = self._input.LA(1) - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class QuotedValueContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def MATCHING_QUOTE_CLOSE(self): - return self.getToken(OmegaConfGrammarParser.MATCHING_QUOTE_CLOSE, 0) - - def QUOTE_OPEN_SINGLE(self): - return self.getToken(OmegaConfGrammarParser.QUOTE_OPEN_SINGLE, 0) - - def QUOTE_OPEN_DOUBLE(self): - return self.getToken(OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE, 0) - - def text(self): - return self.getTypedRuleContext(OmegaConfGrammarParser.TextContext, 0) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_quotedValue - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterQuotedValue"): - listener.enterQuotedValue(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitQuotedValue"): - listener.exitQuotedValue(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitQuotedValue"): - return visitor.visitQuotedValue(self) - else: - return visitor.visitChildren(self) - - def quotedValue(self): - localctx = OmegaConfGrammarParser.QuotedValueContext(self, self._ctx, self.state) - self.enterRule(localctx, 26, self.RULE_quotedValue) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 155 - _la = self._input.LA(1) - if not ( - _la == OmegaConfGrammarParser.QUOTE_OPEN_SINGLE - or _la == OmegaConfGrammarParser.QUOTE_OPEN_DOUBLE - ): - self._errHandler.recoverInline(self) - else: - self._errHandler.reportMatch(self) - self.consume() - self.state = 157 - self._errHandler.sync(self) - _la = self._input.LA(1) - if ((_la) & ~0x3F) == 0 and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.ANY_STR) - | (1 << OmegaConfGrammarParser.ESC_INTER) - | (1 << OmegaConfGrammarParser.TOP_ESC) - | (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.QUOTED_ESC) - ) - ) != 0: - self.state = 156 - self.text() - - self.state = 159 - self.match(OmegaConfGrammarParser.MATCHING_QUOTE_CLOSE) - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class PrimitiveContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def ID(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ID) - else: - return self.getToken(OmegaConfGrammarParser.ID, i) - - def NULL(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.NULL) - else: - return self.getToken(OmegaConfGrammarParser.NULL, i) - - def INT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.INT) - else: - return self.getToken(OmegaConfGrammarParser.INT, i) - - def FLOAT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.FLOAT) - else: - return self.getToken(OmegaConfGrammarParser.FLOAT, i) - - def BOOL(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.BOOL) - else: - return self.getToken(OmegaConfGrammarParser.BOOL, i) - - def UNQUOTED_CHAR(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.UNQUOTED_CHAR) - else: - return self.getToken(OmegaConfGrammarParser.UNQUOTED_CHAR, i) - - def COLON(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.COLON) - else: - return self.getToken(OmegaConfGrammarParser.COLON, i) - - def ESC(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ESC) - else: - return self.getToken(OmegaConfGrammarParser.ESC, i) - - def WS(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.WS) - else: - return self.getToken(OmegaConfGrammarParser.WS, i) - - def interpolation(self, i: int = None): - if i is None: - return self.getTypedRuleContexts(OmegaConfGrammarParser.InterpolationContext) - else: - return self.getTypedRuleContext(OmegaConfGrammarParser.InterpolationContext, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_primitive - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterPrimitive"): - listener.enterPrimitive(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitPrimitive"): - listener.exitPrimitive(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitPrimitive"): - return visitor.visitPrimitive(self) - else: - return visitor.visitChildren(self) - - def primitive(self): - localctx = OmegaConfGrammarParser.PrimitiveContext(self, self._ctx, self.state) - self.enterRule(localctx, 28, self.RULE_primitive) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 171 - self._errHandler.sync(self) - _la = self._input.LA(1) - while True: - self.state = 171 - self._errHandler.sync(self) - token = self._input.LA(1) - if token in [OmegaConfGrammarParser.ID]: - self.state = 161 - self.match(OmegaConfGrammarParser.ID) - pass - elif token in [OmegaConfGrammarParser.NULL]: - self.state = 162 - self.match(OmegaConfGrammarParser.NULL) - pass - elif token in [OmegaConfGrammarParser.INT]: - self.state = 163 - self.match(OmegaConfGrammarParser.INT) - pass - elif token in [OmegaConfGrammarParser.FLOAT]: - self.state = 164 - self.match(OmegaConfGrammarParser.FLOAT) - pass - elif token in [OmegaConfGrammarParser.BOOL]: - self.state = 165 - self.match(OmegaConfGrammarParser.BOOL) - pass - elif token in [OmegaConfGrammarParser.UNQUOTED_CHAR]: - self.state = 166 - self.match(OmegaConfGrammarParser.UNQUOTED_CHAR) - pass - elif token in [OmegaConfGrammarParser.COLON]: - self.state = 167 - self.match(OmegaConfGrammarParser.COLON) - pass - elif token in [OmegaConfGrammarParser.ESC]: - self.state = 168 - self.match(OmegaConfGrammarParser.ESC) - pass - elif token in [OmegaConfGrammarParser.WS]: - self.state = 169 - self.match(OmegaConfGrammarParser.WS) - pass - elif token in [OmegaConfGrammarParser.INTER_OPEN]: - self.state = 170 - self.interpolation() - pass - else: - raise NoViableAltException(self) - - self.state = 173 - self._errHandler.sync(self) - _la = self._input.LA(1) - if not ( - ( - ((_la) & ~0x3F) == 0 - and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.INTER_OPEN) - | (1 << OmegaConfGrammarParser.COLON) - | (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) - != 0 - ) - ): - break - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - class DictKeyContext(ParserRuleContext): - __slots__ = "parser" - - def __init__(self, parser, parent: ParserRuleContext = None, invokingState: int = -1): - super().__init__(parent, invokingState) - self.parser = parser - - def ID(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ID) - else: - return self.getToken(OmegaConfGrammarParser.ID, i) - - def NULL(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.NULL) - else: - return self.getToken(OmegaConfGrammarParser.NULL, i) - - def INT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.INT) - else: - return self.getToken(OmegaConfGrammarParser.INT, i) - - def FLOAT(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.FLOAT) - else: - return self.getToken(OmegaConfGrammarParser.FLOAT, i) - - def BOOL(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.BOOL) - else: - return self.getToken(OmegaConfGrammarParser.BOOL, i) - - def UNQUOTED_CHAR(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.UNQUOTED_CHAR) - else: - return self.getToken(OmegaConfGrammarParser.UNQUOTED_CHAR, i) - - def ESC(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.ESC) - else: - return self.getToken(OmegaConfGrammarParser.ESC, i) - - def WS(self, i: int = None): - if i is None: - return self.getTokens(OmegaConfGrammarParser.WS) - else: - return self.getToken(OmegaConfGrammarParser.WS, i) - - def getRuleIndex(self): - return OmegaConfGrammarParser.RULE_dictKey - - def enterRule(self, listener: ParseTreeListener): - if hasattr(listener, "enterDictKey"): - listener.enterDictKey(self) - - def exitRule(self, listener: ParseTreeListener): - if hasattr(listener, "exitDictKey"): - listener.exitDictKey(self) - - def accept(self, visitor: ParseTreeVisitor): - if hasattr(visitor, "visitDictKey"): - return visitor.visitDictKey(self) - else: - return visitor.visitChildren(self) - - def dictKey(self): - localctx = OmegaConfGrammarParser.DictKeyContext(self, self._ctx, self.state) - self.enterRule(localctx, 30, self.RULE_dictKey) - self._la = 0 # Token type - try: - self.enterOuterAlt(localctx, 1) - self.state = 176 - self._errHandler.sync(self) - _la = self._input.LA(1) - while True: - self.state = 175 - _la = self._input.LA(1) - if not ( - ( - ((_la) & ~0x3F) == 0 - and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) - != 0 - ) - ): - self._errHandler.recoverInline(self) - else: - self._errHandler.reportMatch(self) - self.consume() - self.state = 178 - self._errHandler.sync(self) - _la = self._input.LA(1) - if not ( - ( - ((_la) & ~0x3F) == 0 - and ( - (1 << _la) - & ( - (1 << OmegaConfGrammarParser.FLOAT) - | (1 << OmegaConfGrammarParser.INT) - | (1 << OmegaConfGrammarParser.BOOL) - | (1 << OmegaConfGrammarParser.NULL) - | (1 << OmegaConfGrammarParser.UNQUOTED_CHAR) - | (1 << OmegaConfGrammarParser.ID) - | (1 << OmegaConfGrammarParser.ESC) - | (1 << OmegaConfGrammarParser.WS) - ) - ) - != 0 - ) - ): - break - - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens b/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens deleted file mode 100644 index c22963b90..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarParser.tokens +++ /dev/null @@ -1,31 +0,0 @@ -ANY_STR=1 -ESC_INTER=2 -TOP_ESC=3 -INTER_OPEN=4 -BRACE_OPEN=5 -BRACE_CLOSE=6 -QUOTE_OPEN_SINGLE=7 -QUOTE_OPEN_DOUBLE=8 -COMMA=9 -BRACKET_OPEN=10 -BRACKET_CLOSE=11 -COLON=12 -FLOAT=13 -INT=14 -BOOL=15 -NULL=16 -UNQUOTED_CHAR=17 -ID=18 -ESC=19 -WS=20 -INTER_CLOSE=21 -DOT=22 -INTER_KEY=23 -MATCHING_QUOTE_CLOSE=24 -QUOTED_ESC=25 -DOLLAR=26 -INTER_BRACKET_OPEN=27 -INTER_BRACKET_CLOSE=28 -'.'=22 -'['=27 -']'=28 diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py b/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py deleted file mode 100644 index 282d60ac6..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarParserListener.py +++ /dev/null @@ -1,141 +0,0 @@ -# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 -from antlr4 import * - -if __name__ is not None and "." in __name__: - from .OmegaConfGrammarParser import OmegaConfGrammarParser -else: - from OmegaConfGrammarParser import OmegaConfGrammarParser - - -# This class defines a complete listener for a parse tree produced by OmegaConfGrammarParser. -class OmegaConfGrammarParserListener(ParseTreeListener): - # Enter a parse tree produced by OmegaConfGrammarParser#configValue. - def enterConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#configValue. - def exitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#singleElement. - def enterSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#singleElement. - def exitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#text. - def enterText(self, ctx: OmegaConfGrammarParser.TextContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#text. - def exitText(self, ctx: OmegaConfGrammarParser.TextContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#element. - def enterElement(self, ctx: OmegaConfGrammarParser.ElementContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#element. - def exitElement(self, ctx: OmegaConfGrammarParser.ElementContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#listContainer. - def enterListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#listContainer. - def exitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#dictContainer. - def enterDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#dictContainer. - def exitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. - def enterDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. - def exitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#sequence. - def enterSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#sequence. - def exitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#interpolation. - def enterInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#interpolation. - def exitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#interpolationNode. - def enterInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#interpolationNode. - def exitInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#interpolationResolver. - def enterInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#interpolationResolver. - def exitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#configKey. - def enterConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#configKey. - def exitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#resolverName. - def enterResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#resolverName. - def exitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#quotedValue. - def enterQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#quotedValue. - def exitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#primitive. - def enterPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#primitive. - def exitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): - pass - - # Enter a parse tree produced by OmegaConfGrammarParser#dictKey. - def enterDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): - pass - - # Exit a parse tree produced by OmegaConfGrammarParser#dictKey. - def exitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): - pass - - -del OmegaConfGrammarParser diff --git a/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py b/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py deleted file mode 100644 index 30e15f0b9..000000000 --- a/omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py +++ /dev/null @@ -1,78 +0,0 @@ -# Generated from /home/matteo/github/nexenio/kapitan/omegaconf/omegaconf/grammar/OmegaConfGrammarParser.g4 by ANTLR 4.9.3 -from antlr4 import * - -if __name__ is not None and "." in __name__: - from .OmegaConfGrammarParser import OmegaConfGrammarParser -else: - from OmegaConfGrammarParser import OmegaConfGrammarParser - -# This class defines a complete generic visitor for a parse tree produced by OmegaConfGrammarParser. - - -class OmegaConfGrammarParserVisitor(ParseTreeVisitor): - # Visit a parse tree produced by OmegaConfGrammarParser#configValue. - def visitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#singleElement. - def visitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#text. - def visitText(self, ctx: OmegaConfGrammarParser.TextContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#element. - def visitElement(self, ctx: OmegaConfGrammarParser.ElementContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#listContainer. - def visitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#dictContainer. - def visitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#dictKeyValuePair. - def visitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#sequence. - def visitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#interpolation. - def visitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#interpolationNode. - def visitInterpolationNode(self, ctx: OmegaConfGrammarParser.InterpolationNodeContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#interpolationResolver. - def visitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#configKey. - def visitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#resolverName. - def visitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#quotedValue. - def visitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#primitive. - def visitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by OmegaConfGrammarParser#dictKey. - def visitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext): - return self.visitChildren(ctx) - - -del OmegaConfGrammarParser diff --git a/omegaconf/grammar/gen/__init__.py b/omegaconf/grammar/gen/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/omegaconf/grammar_parser.py b/omegaconf/grammar_parser.py deleted file mode 100644 index 72d5d7cca..000000000 --- a/omegaconf/grammar_parser.py +++ /dev/null @@ -1,140 +0,0 @@ -import re -import threading -from typing import Any - -from antlr4 import CommonTokenStream, InputStream, ParserRuleContext -from antlr4.error.ErrorListener import ErrorListener - -from .errors import GrammarParseError - -# Import from visitor in order to check the presence of generated grammar files -# files in a single place. -from .grammar_visitor import ( # type: ignore - OmegaConfGrammarLexer, - OmegaConfGrammarParser, -) - -# Used to cache grammar objects to avoid re-creating them on each call to `parse()`. -# We use a per-thread cache to make it thread-safe. -_grammar_cache = threading.local() - -# Build regex pattern to efficiently identify typical interpolations. -# See test `test_match_simple_interpolation_pattern` for examples. -_config_key = r"[$\w]+" # foo, $0, $bar, $foo_$bar123$ -_key_maybe_brackets = f"{_config_key}|\\[{_config_key}\\]" # foo, [foo], [$bar] -_node_access = f"\\.{_key_maybe_brackets}" # .foo, [foo], [$bar] -_node_path = f"(\\.)*({_key_maybe_brackets})({_node_access})*" # [foo].bar, .foo[bar] -_node_inter = f"\\${{\\s*{_node_path}\\s*}}" # node interpolation ${foo.bar} -_id = "[a-zA-Z_][\\w\\-]*" # foo, foo_bar, foo-bar, abc123 -_resolver_name = f"({_id}(\\.{_id})*)?" # foo, ns.bar3, ns_1.ns_2.b0z -_arg = r"[a-zA-Z_0-9/\-\+.$%*@?|]+" # string representing a resolver argument -_args = f"{_arg}(\\s*,\\s*{_arg})*" # list of resolver arguments -_resolver_inter = f"\\${{\\s*{_resolver_name}\\s*:\\s*{_args}?\\s*}}" # ${foo:bar} -_inter = f"({_node_inter}|{_resolver_inter})" # any kind of interpolation -_outer = "([^$]|\\$(?!{))+" # any character except $ (unless not followed by {) -SIMPLE_INTERPOLATION_PATTERN = re.compile(f"({_outer})?({_inter}({_outer})?)+$", flags=re.ASCII) -# NOTE: SIMPLE_INTERPOLATION_PATTERN must not generate false positive matches: -# it must not accept anything that isn't a valid interpolation (per the -# interpolation grammar defined in `omegaconf/grammar/*.g4`). - - -class OmegaConfErrorListener(ErrorListener): # type: ignore - def syntaxError( - self, - recognizer: Any, - offending_symbol: Any, - line: Any, - column: Any, - msg: Any, - e: Any, - ) -> None: - raise GrammarParseError(str(e) if msg is None else msg) from e - - def reportAmbiguity( - self, - recognizer: Any, - dfa: Any, - startIndex: Any, - stopIndex: Any, - exact: Any, - ambigAlts: Any, - configs: Any, - ) -> None: - raise GrammarParseError("ANTLR error: Ambiguity") # pragma: no cover - - def reportAttemptingFullContext( - self, - recognizer: Any, - dfa: Any, - startIndex: Any, - stopIndex: Any, - conflictingAlts: Any, - configs: Any, - ) -> None: - # Note: for now we raise an error to be safe. However this is mostly a - # performance warning, so in the future this may be relaxed if we need - # to change the grammar in such a way that this warning cannot be - # avoided (another option would be to switch to SLL parsing mode). - raise GrammarParseError("ANTLR error: Attempting Full Context") # pragma: no cover - - def reportContextSensitivity( - self, - recognizer: Any, - dfa: Any, - startIndex: Any, - stopIndex: Any, - prediction: Any, - configs: Any, - ) -> None: - raise GrammarParseError("ANTLR error: ContextSensitivity") # pragma: no cover - - -def parse( - value: str, parser_rule: str = "configValue", lexer_mode: str = "DEFAULT_MODE" -) -> ParserRuleContext: - """ - Parse interpolated string `value` (and return the parse tree). - """ - l_mode = getattr(OmegaConfGrammarLexer, lexer_mode) - istream = InputStream(value) - - cached = getattr(_grammar_cache, "data", None) - if cached is None: - error_listener = OmegaConfErrorListener() - lexer = OmegaConfGrammarLexer(istream) - lexer.removeErrorListeners() - lexer.addErrorListener(error_listener) - lexer.mode(l_mode) - token_stream = CommonTokenStream(lexer) - parser = OmegaConfGrammarParser(token_stream) - parser.removeErrorListeners() - parser.addErrorListener(error_listener) - - # The two lines below could be enabled in the future if we decide to switch - # to SLL prediction mode. Warning though, it has not been fully tested yet! - # from antlr4 import PredictionMode - # parser._interp.predictionMode = PredictionMode.SLL - - # Note that although the input stream `istream` is implicitly cached within - # the lexer, it will be replaced by a new input next time the lexer is re-used. - _grammar_cache.data = lexer, token_stream, parser - - else: - lexer, token_stream, parser = cached - # Replace the old input stream with the new one. - lexer.inputStream = istream - # Initialize the lexer / token stream / parser to process the new input. - lexer.mode(l_mode) - token_stream.setTokenSource(lexer) - parser.reset() - - try: - return getattr(parser, parser_rule)() - except Exception as exc: - if type(exc) is Exception and str(exc) == "Empty Stack": - # This exception is raised by antlr when trying to pop a mode while - # no mode has been pushed. We convert it into an `GrammarParseError` - # to facilitate exception handling from the caller. - raise GrammarParseError("Empty Stack") - else: - raise diff --git a/omegaconf/grammar_visitor.py b/omegaconf/grammar_visitor.py deleted file mode 100644 index 3c732507f..000000000 --- a/omegaconf/grammar_visitor.py +++ /dev/null @@ -1,362 +0,0 @@ -import sys -import warnings -from itertools import zip_longest -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - List, - Optional, - Set, - Tuple, - Union, -) - -from antlr4 import TerminalNode - -from .errors import InterpolationResolutionError - -if TYPE_CHECKING: - from .base import Node # noqa F401 - -try: - from omegaconf.grammar.gen.OmegaConfGrammarLexer import OmegaConfGrammarLexer - from omegaconf.grammar.gen.OmegaConfGrammarParser import OmegaConfGrammarParser - from omegaconf.grammar.gen.OmegaConfGrammarParserVisitor import ( - OmegaConfGrammarParserVisitor, - ) - -except ModuleNotFoundError: # pragma: no cover - print( - "Error importing OmegaConf's generated parsers, run `python setup.py antlr` to regenerate.", - file=sys.stderr, - ) - sys.exit(1) - - -class GrammarVisitor(OmegaConfGrammarParserVisitor): - def __init__( - self, - node_interpolation_callback: Callable[ - [str, Optional[Set[int]]], - Optional["Node"], - ], - resolver_interpolation_callback: Callable[..., Any], - memo: Optional[Set[int]], - **kw: Dict[Any, Any], - ): - """ - Constructor. - - :param node_interpolation_callback: Callback function that is called when - needing to resolve a node interpolation. This function should take a single - string input which is the key's dot path (ex: `"foo.bar"`). - - :param resolver_interpolation_callback: Callback function that is called when - needing to resolve a resolver interpolation. This function should accept - three keyword arguments: `name` (str, the name of the resolver), - `args` (tuple, the inputs to the resolver), and `args_str` (tuple, - the string representation of the inputs to the resolver). - - :param kw: Additional keyword arguments to be forwarded to parent class. - """ - super().__init__(**kw) - self.node_interpolation_callback = node_interpolation_callback - self.resolver_interpolation_callback = resolver_interpolation_callback - self.memo = memo - - def aggregateResult(self, aggregate: List[Any], nextResult: Any) -> List[Any]: - raise NotImplementedError - - def defaultResult(self) -> List[Any]: - # Raising an exception because not currently used (like `aggregateResult()`). - raise NotImplementedError - - def visitConfigKey(self, ctx: OmegaConfGrammarParser.ConfigKeyContext) -> str: - from ._utils import _get_value - - # interpolation | ID | INTER_KEY - assert ctx.getChildCount() == 1 - child = ctx.getChild(0) - if isinstance(child, OmegaConfGrammarParser.InterpolationContext): - res = _get_value(self.visitInterpolation(child)) - if not isinstance(res, str): - raise InterpolationResolutionError( - f"The following interpolation is used to denote a config key and " - f"thus should return a string, but instead returned `{res}` of " - f"type `{type(res)}`: {ctx.getChild(0).getText()}" - ) - return res - else: - assert isinstance(child, TerminalNode) and isinstance(child.symbol.text, str) - return child.symbol.text - - def visitConfigValue(self, ctx: OmegaConfGrammarParser.ConfigValueContext) -> Any: - # text EOF - assert ctx.getChildCount() == 2 - return self.visit(ctx.getChild(0)) - - def visitDictKey(self, ctx: OmegaConfGrammarParser.DictKeyContext) -> Any: - return self._createPrimitive(ctx) - - def visitDictContainer(self, ctx: OmegaConfGrammarParser.DictContainerContext) -> Dict[Any, Any]: - # BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE - assert ctx.getChildCount() >= 2 - return dict(self.visitDictKeyValuePair(ctx.getChild(i)) for i in range(1, ctx.getChildCount() - 1, 2)) - - def visitElement(self, ctx: OmegaConfGrammarParser.ElementContext) -> Any: - # primitive | quotedValue | listContainer | dictContainer - assert ctx.getChildCount() == 1 - return self.visit(ctx.getChild(0)) - - def visitInterpolation(self, ctx: OmegaConfGrammarParser.InterpolationContext) -> Any: - assert ctx.getChildCount() == 1 # interpolationNode | interpolationResolver - return self.visit(ctx.getChild(0)) - - def visitInterpolationNode( - self, ctx: OmegaConfGrammarParser.InterpolationNodeContext - ) -> Optional["Node"]: - # INTER_OPEN - # DOT* // relative interpolation? - # (configKey | BRACKET_OPEN configKey BRACKET_CLOSE) // foo, [foo] - # (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)* // .foo, [foo], .foo[bar], [foo].bar[baz] - # INTER_CLOSE; - - assert ctx.getChildCount() >= 3 - - inter_key_tokens = [] # parsed elements of the dot path - for child in ctx.getChildren(): - if isinstance(child, TerminalNode): - s = child.symbol - if s.type in [ - OmegaConfGrammarLexer.DOT, - OmegaConfGrammarLexer.BRACKET_OPEN, - OmegaConfGrammarLexer.BRACKET_CLOSE, - ]: - inter_key_tokens.append(s.text) - else: - assert s.type in ( - OmegaConfGrammarLexer.INTER_OPEN, - OmegaConfGrammarLexer.INTER_CLOSE, - ) - else: - assert isinstance(child, OmegaConfGrammarParser.ConfigKeyContext) - inter_key_tokens.append(self.visitConfigKey(child)) - - inter_key = "".join(inter_key_tokens) - return self.node_interpolation_callback(inter_key, self.memo) - - def visitInterpolationResolver(self, ctx: OmegaConfGrammarParser.InterpolationResolverContext) -> Any: - # INTER_OPEN resolverName COLON sequence? BRACE_CLOSE - assert 4 <= ctx.getChildCount() <= 5 - - resolver_name = self.visit(ctx.getChild(1)) - maybe_seq = ctx.getChild(3) - args = [] - args_str = [] - if isinstance(maybe_seq, TerminalNode): # means there are no args - assert maybe_seq.symbol.type == OmegaConfGrammarLexer.BRACE_CLOSE - else: - assert isinstance(maybe_seq, OmegaConfGrammarParser.SequenceContext) - for val, txt in self.visitSequence(maybe_seq): - args.append(val) - args_str.append(txt) - - return self.resolver_interpolation_callback( - name=resolver_name, - args=tuple(args), - args_str=tuple(args_str), - ) - - def visitDictKeyValuePair(self, ctx: OmegaConfGrammarParser.DictKeyValuePairContext) -> Tuple[Any, Any]: - from ._utils import _get_value - - assert ctx.getChildCount() == 3 # dictKey COLON element - key = self.visit(ctx.getChild(0)) - colon = ctx.getChild(1) - assert isinstance(colon, TerminalNode) and colon.symbol.type == OmegaConfGrammarLexer.COLON - value = _get_value(self.visitElement(ctx.getChild(2))) - return key, value - - def visitListContainer(self, ctx: OmegaConfGrammarParser.ListContainerContext) -> List[Any]: - # BRACKET_OPEN sequence? BRACKET_CLOSE; - assert ctx.getChildCount() in (2, 3) - if ctx.getChildCount() == 2: - return [] - sequence = ctx.getChild(1) - assert isinstance(sequence, OmegaConfGrammarParser.SequenceContext) - return list(val for val, _ in self.visitSequence(sequence)) # ignore raw text - - def visitPrimitive(self, ctx: OmegaConfGrammarParser.PrimitiveContext) -> Any: - return self._createPrimitive(ctx) - - def visitQuotedValue(self, ctx: OmegaConfGrammarParser.QuotedValueContext) -> str: - # (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) text? MATCHING_QUOTE_CLOSE - n = ctx.getChildCount() - assert n in [2, 3] - return str(self.visit(ctx.getChild(1))) if n == 3 else "" - - def visitResolverName(self, ctx: OmegaConfGrammarParser.ResolverNameContext) -> str: - from ._utils import _get_value - - # (interpolation | ID) (DOT (interpolation | ID))* - assert ctx.getChildCount() >= 1 - items = [] - for child in list(ctx.getChildren())[::2]: - if isinstance(child, TerminalNode): - assert child.symbol.type == OmegaConfGrammarLexer.ID - items.append(child.symbol.text) - else: - assert isinstance(child, OmegaConfGrammarParser.InterpolationContext) - item = _get_value(self.visitInterpolation(child)) - if not isinstance(item, str): - raise InterpolationResolutionError( - f"The name of a resolver must be a string, but the interpolation " - f"{child.getText()} resolved to `{item}` which is of type " - f"{type(item)}" - ) - items.append(item) - return ".".join(items) - - def visitSequence(self, ctx: OmegaConfGrammarParser.SequenceContext) -> Generator[Any, None, None]: - from ._utils import _get_value - - # (element (COMMA element?)*) | (COMMA element?)+ - assert ctx.getChildCount() >= 1 - - # DEPRECATED: remove in 2.2 (revert #571) - def empty_str_warning() -> None: - txt = ctx.getText() - warnings.warn( - f"In the sequence `{txt}` some elements are missing: please replace " - f"them with empty quoted strings. " - f"See https://github.com/omry/omegaconf/issues/572 for details.", - category=UserWarning, - ) - - is_previous_comma = True # whether previous child was a comma (init to True) - for child in ctx.getChildren(): - if isinstance(child, OmegaConfGrammarParser.ElementContext): - # Also preserve the original text representation of `child` so - # as to allow backward compatibility with old resolvers (registered - # with `legacy_register_resolver()`). Note that we cannot just cast - # the value to string later as for instance `null` would become "None". - yield _get_value(self.visitElement(child)), child.getText() - is_previous_comma = False - else: - assert isinstance(child, TerminalNode) and child.symbol.type == OmegaConfGrammarLexer.COMMA - if is_previous_comma: - empty_str_warning() - yield "", "" - else: - is_previous_comma = True - if is_previous_comma: - # Trailing comma. - empty_str_warning() - yield "", "" - - def visitSingleElement(self, ctx: OmegaConfGrammarParser.SingleElementContext) -> Any: - # element EOF - assert ctx.getChildCount() == 2 - return self.visit(ctx.getChild(0)) - - def visitText(self, ctx: OmegaConfGrammarParser.TextContext) -> Any: - # (interpolation | ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+ - - # Single interpolation? If yes, return its resolved value "as is". - if ctx.getChildCount() == 1: - c = ctx.getChild(0) - if isinstance(c, OmegaConfGrammarParser.InterpolationContext): - return self.visitInterpolation(c) - - # Otherwise, concatenate string representations together. - return self._unescape(list(ctx.getChildren())) - - def _createPrimitive( - self, - ctx: Union[ - OmegaConfGrammarParser.PrimitiveContext, - OmegaConfGrammarParser.DictKeyContext, - ], - ) -> Any: - # (ID | NULL | INT | FLOAT | BOOL | UNQUOTED_CHAR | COLON | ESC | WS | interpolation)+ - if ctx.getChildCount() == 1: - child = ctx.getChild(0) - if isinstance(child, OmegaConfGrammarParser.InterpolationContext): - return self.visitInterpolation(child) - assert isinstance(child, TerminalNode) - symbol = child.symbol - # Parse primitive types. - if symbol.type in ( - OmegaConfGrammarLexer.ID, - OmegaConfGrammarLexer.UNQUOTED_CHAR, - OmegaConfGrammarLexer.COLON, - ): - return symbol.text - elif symbol.type == OmegaConfGrammarLexer.NULL: - return None - elif symbol.type == OmegaConfGrammarLexer.INT: - return int(symbol.text) - elif symbol.type == OmegaConfGrammarLexer.FLOAT: - return float(symbol.text) - elif symbol.type == OmegaConfGrammarLexer.BOOL: - return symbol.text.lower() == "true" - elif symbol.type == OmegaConfGrammarLexer.ESC: - return self._unescape([child]) - elif symbol.type == OmegaConfGrammarLexer.WS: # pragma: no cover - # A single WS should have been "consumed" by another token. - raise AssertionError("WS should never be reached") - assert False, symbol.type - # Concatenation of multiple items ==> un-escape the concatenation. - return self._unescape(list(ctx.getChildren())) - - def _unescape( - self, - seq: List[Union[TerminalNode, OmegaConfGrammarParser.InterpolationContext]], - ) -> str: - """ - Concatenate all symbols / interpolations in `seq`, unescaping symbols as needed. - - Interpolations are resolved and cast to string *WITHOUT* escaping their result - (it is assumed that whatever escaping is required was already handled during the - resolving of the interpolation). - """ - chrs = [] - for node, next_node in zip_longest(seq, seq[1:]): - if isinstance(node, TerminalNode): - s = node.symbol - if s.type == OmegaConfGrammarLexer.ESC_INTER: - # `ESC_INTER` is of the form `\\...\${`: the formula below computes - # the number of characters to keep at the end of the string to remove - # the correct number of backslashes. - text = s.text[-(len(s.text) // 2 + 1) :] - elif ( - # Character sequence identified as requiring un-escaping. - s.type == OmegaConfGrammarLexer.ESC - or ( - # At top level, we need to un-escape backslashes that precede - # an interpolation. - s.type == OmegaConfGrammarLexer.TOP_ESC - and isinstance(next_node, OmegaConfGrammarParser.InterpolationContext) - ) - or ( - # In a quoted sring, we need to un-escape backslashes that - # either end the string, or are followed by an interpolation. - s.type == OmegaConfGrammarLexer.QUOTED_ESC - and ( - next_node is None - or isinstance(next_node, OmegaConfGrammarParser.InterpolationContext) - ) - ) - ): - text = s.text[1::2] # un-escape the sequence - else: - text = s.text # keep the original text - else: - assert isinstance(node, OmegaConfGrammarParser.InterpolationContext) - text = str(self.visitInterpolation(node)) - chrs.append(text) - - return "".join(chrs) diff --git a/omegaconf/listconfig.py b/omegaconf/listconfig.py deleted file mode 100644 index b0f9a4422..000000000 --- a/omegaconf/listconfig.py +++ /dev/null @@ -1,643 +0,0 @@ -import copy -import itertools -from typing import ( - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - MutableSequence, - Optional, - Tuple, - Type, - Union, -) - -from ._utils import ( - ValueKind, - _is_missing_literal, - _is_none, - _resolve_optional, - format_and_raise, - get_value_kind, - is_int, - is_primitive_list, - is_structured_config, - type_str, -) -from .base import Box, ContainerMetadata, Node -from .basecontainer import BaseContainer -from .errors import ( - ConfigAttributeError, - ConfigTypeError, - ConfigValueError, - KeyValidationError, - MissingMandatoryValue, - ReadonlyConfigError, - ValidationError, -) - - -class ListConfig(BaseContainer, MutableSequence[Any]): - _content: Union[List[Node], None, str] - - def __init__( - self, - content: Union[List[Any], Tuple[Any, ...], "ListConfig", str, None], - key: Any = None, - parent: Optional[Box] = None, - element_type: Union[Type[Any], Any] = Any, - is_optional: bool = True, - ref_type: Union[Type[Any], Any] = Any, - flags: Optional[Dict[str, bool]] = None, - ) -> None: - try: - if isinstance(content, ListConfig): - if flags is None: - flags = content._metadata.flags - super().__init__( - parent=parent, - metadata=ContainerMetadata( - ref_type=ref_type, - object_type=list, - key=key, - optional=is_optional, - element_type=element_type, - key_type=int, - flags=flags, - ), - ) - - if isinstance(content, ListConfig): - metadata = copy.deepcopy(content._metadata) - metadata.key = key - metadata.ref_type = ref_type - metadata.optional = is_optional - metadata.element_type = element_type - self.__dict__["_metadata"] = metadata - self._set_value(value=content, flags=flags) - except Exception as ex: - format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex)) - - def _validate_get(self, key: Any, value: Any = None) -> None: - if not isinstance(key, (int, slice)): - raise KeyValidationError("ListConfig indices must be integers or slices, not $KEY_TYPE") - - def _validate_set(self, key: Any, value: Any) -> None: - from omegaconf import OmegaConf - - self._validate_get(key, value) - - if self._get_flag("readonly"): - raise ReadonlyConfigError("ListConfig is read-only") - - if 0 <= key < self.__len__(): - target = self._get_node(key) - if target is not None: - assert isinstance(target, Node) - if value is None and not target._is_optional(): - raise ValidationError("$FULL_KEY is not optional and cannot be assigned None") - - vk = get_value_kind(value) - if vk == ValueKind.MANDATORY_MISSING: - return - else: - is_optional, target_type = _resolve_optional(self._metadata.element_type) - value_type = OmegaConf.get_type(value) - - if (value_type is None and not is_optional) or ( - is_structured_config(target_type) - and value_type is not None - and not issubclass(value_type, target_type) - ): - msg = ( - f"Invalid type assigned: {type_str(value_type)} is not a " - f"subclass of {type_str(target_type)}. value: {value}" - ) - raise ValidationError(msg) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "ListConfig": - res = ListConfig(None) - res.__dict__["_metadata"] = copy.deepcopy(self.__dict__["_metadata"], memo=memo) - res.__dict__["_flags_cache"] = copy.deepcopy(self.__dict__["_flags_cache"], memo=memo) - - src_content = self.__dict__["_content"] - if isinstance(src_content, list): - content_copy: List[Optional[Node]] = [] - for v in src_content: - old_parent = v.__dict__["_parent"] - try: - v.__dict__["_parent"] = None - vc = copy.deepcopy(v, memo=memo) - vc.__dict__["_parent"] = res - content_copy.append(vc) - finally: - v.__dict__["_parent"] = old_parent - else: - # None and strings can be assigned as is - content_copy = src_content - - res.__dict__["_content"] = content_copy - res.__dict__["_parent"] = self.__dict__["_parent"] - - return res - - def copy(self) -> "ListConfig": - return copy.copy(self) - - # hide content while inspecting in debugger - def __dir__(self) -> Iterable[str]: - if self._is_missing() or self._is_none(): - return [] - return [str(x) for x in range(0, len(self))] - - def __setattr__(self, key: str, value: Any) -> None: - self._format_and_raise( - key=key, - value=value, - cause=ConfigAttributeError("ListConfig does not support attribute access"), - ) - assert False - - def __getattr__(self, key: str) -> Any: - # PyCharm is sometimes inspecting __members__, be sure to tell it we don't have that. - if key == "__members__": - raise AttributeError() - - if key == "__name__": - raise AttributeError() - - if is_int(key): - return self.__getitem__(int(key)) - else: - self._format_and_raise( - key=key, - value=None, - cause=ConfigAttributeError("ListConfig does not support attribute access"), - ) - - def __getitem__(self, index: Union[int, slice]) -> Any: - try: - if self._is_missing(): - raise MissingMandatoryValue("ListConfig is missing") - self._validate_get(index, None) - if self._is_none(): - raise TypeError("ListConfig object representing None is not subscriptable") - - assert isinstance(self.__dict__["_content"], list) - if isinstance(index, slice): - result = [] - start, stop, step = self._correct_index_params(index) - for slice_idx in itertools.islice(range(0, len(self)), start, stop, step): - val = self._resolve_with_default( - key=slice_idx, value=self.__dict__["_content"][slice_idx] - ) - result.append(val) - if index.step and index.step < 0: - result.reverse() - return result - else: - return self._resolve_with_default(key=index, value=self.__dict__["_content"][index]) - except Exception as e: - self._format_and_raise(key=index, value=None, cause=e) - - def _correct_index_params(self, index: slice) -> Tuple[int, int, int]: - start = index.start - stop = index.stop - step = index.step - if index.start and index.start < 0: - start = self.__len__() + index.start - if index.stop and index.stop < 0: - stop = self.__len__() + index.stop - if index.step and index.step < 0: - step = abs(step) - if start and stop: - if start > stop: - start, stop = stop + 1, start + 1 - else: - start = stop = 0 - elif not start and stop: - start = list(range(self.__len__() - 1, stop, -step))[0] - stop = None - elif start and not stop: - stop = start + 1 - start = (stop - 1) % step - else: - start = (self.__len__() - 1) % step - return start, stop, step - - def _set_at_index(self, index: Union[int, slice], value: Any) -> None: - self._set_item_impl(index, value) - - def __setitem__(self, index: Union[int, slice], value: Any) -> None: - try: - if isinstance(index, slice): - _ = iter(value) # check iterable - self_indices = index.indices(len(self)) - indexes = range(*self_indices) - - # Ensure lengths match for extended slice assignment - if index.step not in (None, 1): - if len(indexes) != len(value): - raise ValueError( - f"attempt to assign sequence of size {len(value)}" - f" to extended slice of size {len(indexes)}" - ) - - # Initialize insertion offsets for empty slices - if len(indexes) == 0: - curr_index = self_indices[0] - 1 - val_i = -1 - - work_copy = self.copy() # For atomicity manipulate a copy - - # Delete and optionally replace non empty slices - only_removed = 0 - for val_i, i in enumerate(indexes): - curr_index = i - only_removed - del work_copy[curr_index] - if val_i < len(value): - work_copy.insert(curr_index, value[val_i]) - else: - only_removed += 1 - - # Insert any remaining input items - for val_i in range(val_i + 1, len(value)): - curr_index += 1 - work_copy.insert(curr_index, value[val_i]) - - # Reinitialize self with work_copy - self.clear() - self.extend(work_copy) - else: - self._set_at_index(index, value) - except Exception as e: - self._format_and_raise(key=index, value=value, cause=e) - - def append(self, item: Any) -> None: - content = self.__dict__["_content"] - index = len(content) - content.append(None) - try: - self._set_item_impl(index, item) - except Exception as e: - del content[index] - self._format_and_raise(key=index, value=item, cause=e) - assert False - - def _update_keys(self) -> None: - for i in range(len(self)): - node = self._get_node(i) - if node is not None: - assert isinstance(node, Node) - node._metadata.key = i - - def insert(self, index: int, item: Any) -> None: - from omegaconf.omegaconf import _maybe_wrap - - try: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot insert into a read-only ListConfig") - if self._is_none(): - raise TypeError("Cannot insert into ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot insert into missing ListConfig") - - try: - assert isinstance(self.__dict__["_content"], list) - # insert place holder - self.__dict__["_content"].insert(index, None) - is_optional, ref_type = _resolve_optional(self._metadata.element_type) - node = _maybe_wrap( - ref_type=ref_type, - key=index, - value=item, - is_optional=is_optional, - parent=self, - ) - self._validate_set(key=index, value=node) - self._set_at_index(index, node) - self._update_keys() - except Exception: - del self.__dict__["_content"][index] - self._update_keys() - raise - except Exception as e: - self._format_and_raise(key=index, value=item, cause=e) - assert False - - def extend(self, lst: Iterable[Any]) -> None: - assert isinstance(lst, (tuple, list, ListConfig)) - for x in lst: - self.append(x) - - def remove(self, x: Any) -> None: - del self[self.index(x)] - - def __delitem__(self, key: Union[int, slice]) -> None: - if self._get_flag("readonly"): - self._format_and_raise( - key=key, - value=None, - cause=ReadonlyConfigError("Cannot delete item from read-only ListConfig"), - ) - del self.__dict__["_content"][key] - self._update_keys() - - def clear(self) -> None: - del self[:] - - def index(self, x: Any, start: Optional[int] = None, end: Optional[int] = None) -> int: - if start is None: - start = 0 - if end is None: - end = len(self) - assert start >= 0 - assert end <= len(self) - found_idx = -1 - for idx in range(start, end): - item = self[idx] - if x == item: - found_idx = idx - break - if found_idx != -1: - return found_idx - else: - self._format_and_raise( - key=None, - value=None, - cause=ConfigValueError("Item not found in ListConfig"), - ) - assert False - - def count(self, x: Any) -> int: - c = 0 - for item in self: - if item == x: - c = c + 1 - return c - - def _get_node( - self, - key: Union[int, slice], - validate_access: bool = True, - validate_key: bool = True, - throw_on_missing_value: bool = False, - throw_on_missing_key: bool = False, - ) -> Union[Optional[Node], List[Optional[Node]]]: - try: - if self._is_none(): - raise TypeError("Cannot get_node from a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot get_node from a missing ListConfig") - assert isinstance(self.__dict__["_content"], list) - if validate_access: - self._validate_get(key) - - value = self.__dict__["_content"][key] - if value is not None: - if isinstance(key, slice): - assert isinstance(value, list) - for v in value: - if throw_on_missing_value and v._is_missing(): - raise MissingMandatoryValue("Missing mandatory value") - else: - assert isinstance(value, Node) - if throw_on_missing_value and value._is_missing(): - raise MissingMandatoryValue("Missing mandatory value: $KEY") - return value - except (IndexError, TypeError, MissingMandatoryValue, KeyValidationError) as e: - if isinstance(e, MissingMandatoryValue) and throw_on_missing_value: - raise - if validate_access: - self._format_and_raise(key=key, value=None, cause=e) - assert False - else: - return None - - def get(self, index: int, default_value: Any = None) -> Any: - try: - if self._is_none(): - raise TypeError("Cannot get from a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot get from a missing ListConfig") - self._validate_get(index, None) - assert isinstance(self.__dict__["_content"], list) - return self._resolve_with_default( - key=index, - value=self.__dict__["_content"][index], - default_value=default_value, - ) - except Exception as e: - self._format_and_raise(key=index, value=None, cause=e) - assert False - - def pop(self, index: int = -1) -> Any: - try: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot pop from read-only ListConfig") - if self._is_none(): - raise TypeError("Cannot pop from a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot pop from a missing ListConfig") - - assert isinstance(self.__dict__["_content"], list) - node = self._get_child(index) - assert isinstance(node, Node) - ret = self._resolve_with_default(key=index, value=node, default_value=None) - del self.__dict__["_content"][index] - self._update_keys() - return ret - except KeyValidationError as e: - self._format_and_raise(key=index, value=None, cause=e, type_override=ConfigTypeError) - assert False - except Exception as e: - self._format_and_raise(key=index, value=None, cause=e) - assert False - - def sort(self, key: Optional[Callable[[Any], Any]] = None, reverse: bool = False) -> None: - try: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot sort a read-only ListConfig") - if self._is_none(): - raise TypeError("Cannot sort a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot sort a missing ListConfig") - - if key is None: - - def key1(x: Any) -> Any: - return x._value() - - else: - - def key1(x: Any) -> Any: - return key(x._value()) # type: ignore - - assert isinstance(self.__dict__["_content"], list) - self.__dict__["_content"].sort(key=key1, reverse=reverse) - - except Exception as e: - self._format_and_raise(key=None, value=None, cause=e) - assert False - - def __eq__(self, other: Any) -> bool: - if isinstance(other, (list, tuple)) or other is None: - other = ListConfig(other, flags={"allow_objects": True}) - return ListConfig._list_eq(self, other) - if other is None or isinstance(other, ListConfig): - return ListConfig._list_eq(self, other) - if self._is_missing(): - return _is_missing_literal(other) - return NotImplemented - - def __ne__(self, other: Any) -> bool: - x = self.__eq__(other) - if x is not NotImplemented: - return not x - return NotImplemented - - def __hash__(self) -> int: - return hash(str(self)) - - def __iter__(self) -> Iterator[Any]: - return self._iter_ex(resolve=True) - - class ListIterator(Iterator[Any]): - def __init__(self, lst: Any, resolve: bool) -> None: - self.resolve = resolve - self.iterator = iter(lst.__dict__["_content"]) - self.index = 0 - from .nodes import ValueNode - - self.ValueNode = ValueNode - - def __next__(self) -> Any: - x = next(self.iterator) - if self.resolve: - x = x._dereference_node() - if x._is_missing(): - raise MissingMandatoryValue(f"Missing value at index {self.index}") - - self.index = self.index + 1 - if isinstance(x, self.ValueNode): - return x._value() - else: - # Must be omegaconf.Container. not checking for perf reasons. - if x._is_none(): - return None - return x - - def __repr__(self) -> str: # pragma: no cover - return f"ListConfig.ListIterator(resolve={self.resolve})" - - def _iter_ex(self, resolve: bool) -> Iterator[Any]: - try: - if self._is_none(): - raise TypeError("Cannot iterate a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot iterate a missing ListConfig") - - return ListConfig.ListIterator(self, resolve) - except (TypeError, MissingMandatoryValue) as e: - self._format_and_raise(key=None, value=None, cause=e) - assert False - - def __add__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig": - # res is sharing this list's parent to allow interpolation to work as expected - res = ListConfig(parent=self._get_parent(), content=[]) - res.extend(self) - res.extend(other) - return res - - def __radd__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig": - # res is sharing this list's parent to allow interpolation to work as expected - res = ListConfig(parent=self._get_parent(), content=[]) - res.extend(other) - res.extend(self) - return res - - def __iadd__(self, other: Iterable[Any]) -> "ListConfig": - self.extend(other) - return self - - def __contains__(self, item: Any) -> bool: - if self._is_none(): - raise TypeError("Cannot check if an item is in a ListConfig object representing None") - if self._is_missing(): - raise MissingMandatoryValue("Cannot check if an item is in missing ListConfig") - - lst = self.__dict__["_content"] - for x in lst: - x = x._dereference_node() - if x == item: - return True - return False - - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - try: - previous_content = self.__dict__["_content"] - previous_metadata = self.__dict__["_metadata"] - self._set_value_impl(value, flags) - except Exception as e: - self.__dict__["_content"] = previous_content - self.__dict__["_metadata"] = previous_metadata - raise e - - def _set_value_impl(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - from omegaconf import MISSING, flag_override - - if flags is None: - flags = {} - - vk = get_value_kind(value, strict_interpolation_validation=True) - if _is_none(value): - if not self._is_optional(): - raise ValidationError("Non optional ListConfig cannot be constructed from None") - self.__dict__["_content"] = None - self._metadata.object_type = None - elif vk is ValueKind.MANDATORY_MISSING: - self.__dict__["_content"] = MISSING - self._metadata.object_type = None - elif vk == ValueKind.INTERPOLATION: - self.__dict__["_content"] = value - self._metadata.object_type = None - else: - if not (is_primitive_list(value) or isinstance(value, ListConfig)): - type_ = type(value) - msg = f"Invalid value assigned: {type_.__name__} is not a ListConfig, list or tuple." - raise ValidationError(msg) - - self.__dict__["_content"] = [] - if isinstance(value, ListConfig): - self._metadata.flags = copy.deepcopy(flags) - # disable struct and readonly for the construction phase - # retaining other flags like allow_objects. The real flags are restored at the end of this function - with flag_override(self, ["struct", "readonly"], False): - for item in value._iter_ex(resolve=False): - self.append(item) - elif is_primitive_list(value): - with flag_override(self, ["struct", "readonly"], False): - for item in value: - self.append(item) - self._metadata.object_type = list - - @staticmethod - def _list_eq(l1: Optional["ListConfig"], l2: Optional["ListConfig"]) -> bool: - l1_none = l1.__dict__["_content"] is None - l2_none = l2.__dict__["_content"] is None - if l1_none and l2_none: - return True - if l1_none != l2_none: - return False - - assert isinstance(l1, ListConfig) - assert isinstance(l2, ListConfig) - if len(l1) != len(l2): - return False - for i in range(len(l1)): - if not BaseContainer._item_eq(l1, i, l2, i): - return False - - return True diff --git a/omegaconf/nodes.py b/omegaconf/nodes.py deleted file mode 100644 index 44ce6a591..000000000 --- a/omegaconf/nodes.py +++ /dev/null @@ -1,513 +0,0 @@ -import copy -import math -import sys -from abc import abstractmethod -from enum import Enum -from pathlib import Path -from typing import Any, Dict, Optional, Type, Union - -from omegaconf._utils import ( - ValueKind, - _is_interpolation, - get_type_of, - get_value_kind, - is_primitive_container, - type_str, -) -from omegaconf.base import Box, DictKeyType, Metadata, Node -from omegaconf.errors import ReadonlyConfigError, UnsupportedValueType, ValidationError - - -class ValueNode(Node): - _val: Any - - def __init__(self, parent: Optional[Box], value: Any, metadata: Metadata): - from omegaconf import read_write - - super().__init__(parent=parent, metadata=metadata) - with read_write(self): - self._set_value(value) # lgtm [py/init-calls-subclass] - - def _value(self) -> Any: - return self._val - - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot set value of read-only config node") - - if isinstance(value, str) and get_value_kind(value, strict_interpolation_validation=True) in ( - ValueKind.INTERPOLATION, - ValueKind.MANDATORY_MISSING, - ): - self._val = value - else: - self._val = self.validate_and_convert(value) - - def _strict_validate_type(self, value: Any) -> None: - ref_type = self._metadata.ref_type - if isinstance(ref_type, type) and type(value) is not ref_type: - type_hint = type_str(self._metadata.type_hint) - raise ValidationError( - f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" - ) - - def validate_and_convert(self, value: Any) -> Any: - """ - Validates input and converts to canonical form - :param value: input value - :return: converted value ("100" may be converted to 100 for example) - """ - if value is None: - if self._is_optional(): - return None - ref_type_str = type_str(self._metadata.ref_type) - raise ValidationError(f"Incompatible value '{value}' for field of type '{ref_type_str}'") - - # Subclasses can assume that `value` is not None in - # `_validate_and_convert_impl()` and in `_strict_validate_type()`. - if self._get_flag("convert") is False: - self._strict_validate_type(value) - return value - else: - return self._validate_and_convert_impl(value) - - @abstractmethod - def _validate_and_convert_impl(self, value: Any) -> Any: - ... - - def __str__(self) -> str: - return str(self._val) - - def __repr__(self) -> str: - return repr(self._val) if hasattr(self, "_val") else "__INVALID__" - - def __eq__(self, other: Any) -> bool: - if isinstance(other, AnyNode): - return self._val == other._val # type: ignore - else: - return self._val == other # type: ignore - - def __ne__(self, other: Any) -> bool: - x = self.__eq__(other) - assert x is not NotImplemented - return not x - - def __hash__(self) -> int: - return hash(self._val) - - def _deepcopy_impl(self, res: Any, memo: Dict[int, Any]) -> None: - res.__dict__["_metadata"] = copy.deepcopy(self._metadata, memo=memo) - # shallow copy for value to support non-copyable value - res.__dict__["_val"] = self._val - - # parent is retained, but not copied - res.__dict__["_parent"] = self._parent - - def _is_optional(self) -> bool: - return self._metadata.optional - - def _is_interpolation(self) -> bool: - return _is_interpolation(self._value()) - - def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: - parent = self._get_parent() - if parent is None: - if self._metadata.key is None: - return "" - else: - return str(self._metadata.key) - else: - return parent._get_full_key(self._metadata.key) - - -class AnyNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata(ref_type=Any, object_type=None, key=key, optional=True, flags=flags), - ) - - def _validate_and_convert_impl(self, value: Any) -> Any: - from ._utils import is_primitive_type_annotation - - # allow_objects is internal and not an official API. use at your own risk. - # Please be aware that this support is subject to change without notice. - # If this is deemed useful and supportable it may become an official API. - - if self._get_flag("allow_objects") is not True and not is_primitive_type_annotation(value): - t = get_type_of(value) - raise UnsupportedValueType(f"Value '{t.__name__}' is not a supported primitive type") - return value - - def __deepcopy__(self, memo: Dict[int, Any]) -> "AnyNode": - res = AnyNode() - self._deepcopy_impl(res, memo) - return res - - -class StringNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=str, - object_type=str, - flags=flags, - ), - ) - - def _validate_and_convert_impl(self, value: Any) -> str: - from omegaconf import OmegaConf - - if OmegaConf.is_config(value) or is_primitive_container(value) or isinstance(value, bytes): - raise ValidationError("Cannot convert '$VALUE_TYPE' to string: '$VALUE'") - return str(value) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "StringNode": - res = StringNode() - self._deepcopy_impl(res, memo) - return res - - -class PathNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=Path, - object_type=Path, - flags=flags, - ), - ) - - def _strict_validate_type(self, value: Any) -> None: - if not isinstance(value, Path): - raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' is not an instance of 'pathlib.Path'") - - def _validate_and_convert_impl(self, value: Any) -> Path: - if not isinstance(value, (str, Path)): - raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Path") - - return Path(value) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "PathNode": - res = PathNode() - self._deepcopy_impl(res, memo) - return res - - -class IntegerNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=int, - object_type=int, - flags=flags, - ), - ) - - def _validate_and_convert_impl(self, value: Any) -> int: - try: - if type(value) in (str, int): - val = int(value) - else: - raise ValueError() - except ValueError: - raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Integer") - return val - - def __deepcopy__(self, memo: Dict[int, Any]) -> "IntegerNode": - res = IntegerNode() - self._deepcopy_impl(res, memo) - return res - - -class BytesNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=bytes, - object_type=bytes, - flags=flags, - ), - ) - - def _validate_and_convert_impl(self, value: Any) -> bytes: - if not isinstance(value, bytes): - raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' is not of type 'bytes'") - return value - - def __deepcopy__(self, memo: Dict[int, Any]) -> "BytesNode": - res = BytesNode() - self._deepcopy_impl(res, memo) - return res - - -class FloatNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=float, - object_type=float, - flags=flags, - ), - ) - - def _validate_and_convert_impl(self, value: Any) -> float: - try: - if type(value) in (float, str, int): - return float(value) - else: - raise ValueError() - except ValueError: - raise ValidationError("Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Float") - - def __eq__(self, other: Any) -> bool: - if isinstance(other, ValueNode): - other_val = other._val - else: - other_val = other - if self._val is None and other is None: - return True - if self._val is None and other is not None: - return False - if self._val is not None and other is None: - return False - nan1 = math.isnan(self._val) if isinstance(self._val, float) else False - nan2 = math.isnan(other_val) if isinstance(other_val, float) else False - return self._val == other_val or (nan1 and nan2) - - def __hash__(self) -> int: - return hash(self._val) - - def __deepcopy__(self, memo: Dict[int, Any]) -> "FloatNode": - res = FloatNode() - self._deepcopy_impl(res, memo) - return res - - -class BooleanNode(ValueNode): - def __init__( - self, - value: Any = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=bool, - object_type=bool, - flags=flags, - ), - ) - - def _validate_and_convert_impl(self, value: Any) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, int): - return value != 0 - elif isinstance(value, str): - try: - return self._validate_and_convert_impl(int(value)) - except ValueError as e: - if value.lower() in ("yes", "y", "on", "true"): - return True - elif value.lower() in ("no", "n", "off", "false"): - return False - else: - raise ValidationError( - "Value '$VALUE' is not a valid bool (type $VALUE_TYPE)" - ).with_traceback(sys.exc_info()[2]) from e - else: - raise ValidationError("Value '$VALUE' is not a valid bool (type $VALUE_TYPE)") - - def __deepcopy__(self, memo: Dict[int, Any]) -> "BooleanNode": - res = BooleanNode() - self._deepcopy_impl(res, memo) - return res - - -class EnumNode(ValueNode): # lgtm [py/missing-equals] : Intentional. - """ - NOTE: EnumNode is serialized to yaml as a string ("Color.BLUE"), not as a fully qualified yaml type. - this means serialization to YAML of a typed config (with EnumNode) will not retain the type of the Enum - when loaded. - This is intentional, Please open an issue against OmegaConf if you wish to discuss this decision. - """ - - def __init__( - self, - enum_type: Type[Enum], - value: Optional[Union[Enum, str]] = None, - key: Any = None, - parent: Optional[Box] = None, - is_optional: bool = True, - flags: Optional[Dict[str, bool]] = None, - ): - if not isinstance(enum_type, type) or not issubclass(enum_type, Enum): - raise ValidationError(f"EnumNode can only operate on Enum subclasses ({enum_type})") - self.fields: Dict[str, str] = {} - self.enum_type: Type[Enum] = enum_type - for name, constant in enum_type.__members__.items(): - self.fields[name] = constant.value - super().__init__( - parent=parent, - value=value, - metadata=Metadata( - key=key, - optional=is_optional, - ref_type=enum_type, - object_type=enum_type, - flags=flags, - ), - ) - - def _strict_validate_type(self, value: Any) -> None: - ref_type = self._metadata.ref_type - if not isinstance(value, ref_type): - type_hint = type_str(self._metadata.type_hint) - raise ValidationError( - f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" - ) - - def _validate_and_convert_impl(self, value: Any) -> Enum: - return self.validate_and_convert_to_enum(enum_type=self.enum_type, value=value) - - @staticmethod - def validate_and_convert_to_enum(enum_type: Type[Enum], value: Any) -> Enum: - if not isinstance(value, (str, int)) and not isinstance(value, enum_type): - raise ValidationError(f"Value $VALUE ($VALUE_TYPE) is not a valid input for {enum_type}") - - if isinstance(value, enum_type): - return value - - try: - if isinstance(value, (float, bool)): - raise ValueError - - if isinstance(value, int): - return enum_type(value) - - if isinstance(value, str): - prefix = f"{enum_type.__name__}." - if value.startswith(prefix): - value = value[len(prefix) :] - return enum_type[value] - - assert False - - except (ValueError, KeyError) as e: - valid = ", ".join([x for x in enum_type.__members__.keys()]) - raise ValidationError(f"Invalid value '$VALUE', expected one of [{valid}]").with_traceback( - sys.exc_info()[2] - ) from e - - def __deepcopy__(self, memo: Dict[int, Any]) -> "EnumNode": - res = EnumNode(enum_type=self.enum_type) - self._deepcopy_impl(res, memo) - return res - - -class InterpolationResultNode(ValueNode): - """ - Special node type, used to wrap interpolation results. - """ - - def __init__( - self, - value: Any, - key: Any = None, - parent: Optional[Box] = None, - flags: Optional[Dict[str, bool]] = None, - ): - super().__init__( - parent=parent, - value=value, - metadata=Metadata(ref_type=Any, object_type=None, key=key, optional=True, flags=flags), - ) - # In general we should not try to write into interpolation results. - if flags is None or "readonly" not in flags: - self._set_flag("readonly", True) - - def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: - if self._get_flag("readonly"): - raise ReadonlyConfigError("Cannot set value of read-only config node") - self._val = self.validate_and_convert(value) - - def _validate_and_convert_impl(self, value: Any) -> Any: - # Interpolation results may be anything. - return value - - def __deepcopy__(self, memo: Dict[int, Any]) -> "InterpolationResultNode": - # Currently there should be no need to deep-copy such nodes. - raise NotImplementedError - - def _is_interpolation(self) -> bool: - # The result of an interpolation cannot be itself an interpolation. - return False diff --git a/omegaconf/omegaconf.py b/omegaconf/omegaconf.py deleted file mode 100644 index e438eb244..000000000 --- a/omegaconf/omegaconf.py +++ /dev/null @@ -1,1144 +0,0 @@ -"""OmegaConf module""" -import copy -import inspect -import io -import os -import pathlib -import sys -import warnings -from collections import defaultdict -from contextlib import contextmanager -from enum import Enum -from textwrap import dedent -from typing import ( - IO, - Any, - Callable, - Dict, - Generator, - Iterable, - List, - Optional, - Set, - Tuple, - Type, - Union, - overload, -) - -import yaml - -from . import DictConfig, DictKeyType, ListConfig -from ._utils import ( - _DEFAULT_MARKER_, - _ensure_container, - _get_value, - format_and_raise, - get_dict_key_value_types, - get_list_element_type, - get_omega_conf_dumper, - get_type_of, - is_attr_class, - is_dataclass, - is_dict_annotation, - is_int, - is_list_annotation, - is_primitive_container, - is_primitive_dict, - is_primitive_list, - is_structured_config, - is_tuple_annotation, - is_union_annotation, - nullcontext, - split_key, - type_str, -) -from .base import Box, Container, Node, SCMode, UnionNode -from .basecontainer import BaseContainer -from .errors import ( - MissingMandatoryValue, - OmegaConfBaseException, - UnsupportedInterpolationType, - ValidationError, -) -from .nodes import ( - AnyNode, - BooleanNode, - BytesNode, - EnumNode, - FloatNode, - IntegerNode, - PathNode, - StringNode, - ValueNode, -) - -MISSING: Any = "???" - -Resolver = Callable[..., Any] - - -def II(interpolation: str) -> Any: - """ - Equivalent to ``${interpolation}`` - - :param interpolation: - :return: input ``${node}`` with type Any - """ - return "${" + interpolation + "}" - - -def SI(interpolation: str) -> Any: - """ - Use this for String interpolation, for example ``"http://${host}:${port}"`` - - :param interpolation: interpolation string - :return: input interpolation with type ``Any`` - """ - return interpolation - - -def register_default_resolvers() -> None: - from omegaconf.resolvers import oc - - OmegaConf.register_new_resolver("oc.create", oc.create) - OmegaConf.register_new_resolver("oc.decode", oc.decode) - OmegaConf.register_new_resolver("oc.deprecated", oc.deprecated) - OmegaConf.register_new_resolver("oc.env", oc.env) - OmegaConf.register_new_resolver("oc.select", oc.select) - OmegaConf.register_new_resolver("oc.dict.keys", oc.dict.keys) - OmegaConf.register_new_resolver("oc.dict.values", oc.dict.values) - - -class OmegaConf: - """OmegaConf primary class""" - - def __init__(self) -> None: - raise NotImplementedError("Use one of the static construction functions") - - @staticmethod - def structured( - obj: Any, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> Any: - return OmegaConf.create(obj, parent, flags) - - @staticmethod - @overload - def create( - obj: str, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> Union[DictConfig, ListConfig]: - ... - - @staticmethod - @overload - def create( - obj: Union[List[Any], Tuple[Any, ...]], - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> ListConfig: - ... - - @staticmethod - @overload - def create( - obj: DictConfig, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> DictConfig: - ... - - @staticmethod - @overload - def create( - obj: ListConfig, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> ListConfig: - ... - - @staticmethod - @overload - def create( - obj: Optional[Dict[Any, Any]] = None, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> DictConfig: - ... - - @staticmethod - def create( # noqa F811 - obj: Any = _DEFAULT_MARKER_, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> Union[DictConfig, ListConfig]: - return OmegaConf._create_impl( - obj=obj, - parent=parent, - flags=flags, - ) - - @staticmethod - def load(file_: Union[str, pathlib.Path, IO[Any]]) -> Union[DictConfig, ListConfig]: - from ._utils import get_yaml_loader - - if isinstance(file_, (str, pathlib.Path)): - with io.open(os.path.abspath(file_), "r", encoding="utf-8") as f: - obj = yaml.load(f, Loader=get_yaml_loader()) - elif getattr(file_, "read", None): - obj = yaml.load(file_, Loader=get_yaml_loader()) - else: - raise TypeError("Unexpected file type") - - if obj is not None and not isinstance(obj, (list, dict, str)): - raise IOError(f"Invalid loaded object type: {type(obj).__name__}") # pragma: no cover - - ret: Union[DictConfig, ListConfig] - if obj is None: - ret = OmegaConf.create() - else: - ret = OmegaConf.create(obj) - return ret - - @staticmethod - def save(config: Any, f: Union[str, pathlib.Path, IO[Any]], resolve: bool = False) -> None: - """ - Save as configuration object to a file - - :param config: omegaconf.Config object (DictConfig or ListConfig). - :param f: filename or file object - :param resolve: True to save a resolved config (defaults to False) - """ - if is_dataclass(config) or is_attr_class(config): - config = OmegaConf.create(config) - data = OmegaConf.to_yaml(config, resolve=resolve) - if isinstance(f, (str, pathlib.Path)): - with io.open(os.path.abspath(f), "w", encoding="utf-8") as file: - file.write(data) - elif hasattr(f, "write"): - f.write(data) - f.flush() - else: - raise TypeError("Unexpected file type") - - @staticmethod - def from_cli(args_list: Optional[List[str]] = None) -> DictConfig: - if args_list is None: - # Skip program name - args_list = sys.argv[1:] - return OmegaConf.from_dotlist(args_list) - - @staticmethod - def from_dotlist(dotlist: List[str]) -> DictConfig: - """ - Creates config from the content sys.argv or from the specified args list of not None - - :param dotlist: A list of dotlist-style strings, e.g. ``["foo.bar=1", "baz=qux"]``. - :return: A ``DictConfig`` object created from the dotlist. - """ - conf = OmegaConf.create() - conf.merge_with_dotlist(dotlist) - return conf - - @staticmethod - def merge( - *configs: Union[ - DictConfig, - ListConfig, - Dict[DictKeyType, Any], - List[Any], - Tuple[Any, ...], - Any, - ], - extend_lists: bool = False, - remove_duplicates: bool = False, - ) -> Union[ListConfig, DictConfig]: - """ - Merge a list of previously created configs into a single one - - :param configs: Input configs - :param extend_lists: flag to deep merge ListConfigs - :param remove_duplicates: lists entries are unique. This flag is ignored unless `extend_lists=True` - :return: the merged config object. - """ - assert len(configs) > 0 - target = copy.deepcopy(configs[0]) - target = _ensure_container(target) - assert isinstance(target, (DictConfig, ListConfig)) - - with flag_override(target, "readonly", False): - target.merge_with( - *configs[1:], - extend_lists=extend_lists, - remove_duplicates=remove_duplicates, - ) - turned_readonly = target._get_flag("readonly") is True - - if turned_readonly: - OmegaConf.set_readonly(target, True) - - return target - - @staticmethod - def unsafe_merge( - *configs: Union[ - DictConfig, - ListConfig, - Dict[DictKeyType, Any], - List[Any], - Tuple[Any, ...], - Any, - ], - extend_lists: bool = False, - remove_duplicates: bool = False, - ) -> Union[ListConfig, DictConfig]: - """ - Merge a list of previously created configs into a single one - This is much faster than OmegaConf.merge() as the input configs are not copied. - However, the input configs must not be used after this operation as will become inconsistent. - - :param configs: Input configs - :param extend_lists: flag to deep merge ListConfigs - :param remove_duplicates: lists entries are unique. This flag is ignored unless `extend_lists=True` - :return: the merged config object. - """ - assert len(configs) > 0 - target = configs[0] - target = _ensure_container(target) - assert isinstance(target, (DictConfig, ListConfig)) - - with flag_override(target, ["readonly", "no_deepcopy_set_nodes"], [False, True]): - target.merge_with( - *configs[1:], - extend_lists=extend_lists, - remove_duplicates=remove_duplicates, - ) - turned_readonly = target._get_flag("readonly") is True - - if turned_readonly: - OmegaConf.set_readonly(target, True) - - return target - - @staticmethod - def register_resolver(name: str, resolver: Resolver) -> None: - warnings.warn( - dedent( - """\ - register_resolver() is deprecated. - See https://github.com/omry/omegaconf/issues/426 for migration instructions. - """ - ), - stacklevel=2, - ) - return OmegaConf.legacy_register_resolver(name, resolver) - - # This function will eventually be deprecated and removed. - @staticmethod - def legacy_register_resolver(name: str, resolver: Resolver) -> None: - assert callable(resolver), "resolver must be callable" - # noinspection PyProtectedMember - assert name not in BaseContainer._resolvers, f"resolver '{name}' is already registered" - - def resolver_wrapper( - config: BaseContainer, - parent: BaseContainer, - node: Node, - args: Tuple[Any, ...], - args_str: Tuple[str, ...], - ) -> Any: - cache = OmegaConf.get_cache(config)[name] - # "Un-escape " spaces and commas. - args_unesc = [x.replace(r"\ ", " ").replace(r"\,", ",") for x in args_str] - - # Nested interpolations behave in a potentially surprising way with - # legacy resolvers (they remain as strings, e.g., "${foo}"). If any - # input looks like an interpolation we thus raise an exception. - try: - bad_arg = next(i for i in args_unesc if "${" in i) - except StopIteration: - pass - else: - raise ValueError( - f"Resolver '{name}' was called with argument '{bad_arg}' that appears " - f"to be an interpolation. Nested interpolations are not supported for " - f"resolvers registered with `[legacy_]register_resolver()`, please use " - f"`register_new_resolver()` instead (see " - f"https://github.com/omry/omegaconf/issues/426 for migration instructions)." - ) - key = args_str - val = cache[key] if key in cache else resolver(*args_unesc) - cache[key] = val - return val - - # noinspection PyProtectedMember - BaseContainer._resolvers[name] = resolver_wrapper - - @staticmethod - def register_new_resolver( - name: str, - resolver: Resolver, - *, - replace: bool = False, - use_cache: bool = False, - ) -> None: - """ - Register a resolver. - - :param name: Name of the resolver. - :param resolver: Callable whose arguments are provided in the interpolation, - e.g., with ${foo:x,0,${y.z}} these arguments are respectively "x" (str), - 0 (int) and the value of ``y.z``. - :param replace: If set to ``False`` (default), then a ``ValueError`` is raised if - an existing resolver has already been registered with the same name. - If set to ``True``, then the new resolver replaces the previous one. - NOTE: The cache on existing config objects is not affected, use - ``OmegaConf.clear_cache(cfg)`` to clear it. - :param use_cache: Whether the resolver's outputs should be cached. The cache is - based only on the string literals representing the resolver arguments, e.g., - ${foo:${bar}} will always return the same value regardless of the value of - ``bar`` if the cache is enabled for ``foo``. - """ - if not callable(resolver): - raise TypeError("resolver must be callable") - if not name: - raise ValueError("cannot use an empty resolver name") - - if not replace and OmegaConf.has_resolver(name): - raise ValueError(f"resolver '{name}' is already registered") - - try: - sig: Optional[inspect.Signature] = inspect.signature(resolver) - except ValueError: - sig = None - - def _should_pass(special: str) -> bool: - ret = sig is not None and special in sig.parameters - if ret and use_cache: - raise ValueError(f"use_cache=True is incompatible with functions that receive the {special}") - return ret - - pass_parent = _should_pass("_parent_") - pass_node = _should_pass("_node_") - pass_root = _should_pass("_root_") - - def resolver_wrapper( - config: BaseContainer, - parent: Container, - node: Node, - args: Tuple[Any, ...], - args_str: Tuple[str, ...], - ) -> Any: - if use_cache: - cache = OmegaConf.get_cache(config)[name] - try: - return cache[args_str] - except KeyError: - pass - - # Call resolver. - kwargs: Dict[str, Node] = {} - if pass_parent: - kwargs["_parent_"] = parent - if pass_node: - kwargs["_node_"] = node - if pass_root: - kwargs["_root_"] = config - - ret = resolver(*args, **kwargs) - - if use_cache: - cache[args_str] = ret - return ret - - # noinspection PyProtectedMember - BaseContainer._resolvers[name] = resolver_wrapper - - @classmethod - def has_resolver(cls, name: str) -> bool: - return cls._get_resolver(name) is not None - - # noinspection PyProtectedMember - @staticmethod - def clear_resolvers() -> None: - """ - Clear(remove) all OmegaConf resolvers, then re-register OmegaConf's default resolvers. - """ - BaseContainer._resolvers = {} - register_default_resolvers() - - @classmethod - def clear_resolver(cls, name: str) -> bool: - """ - Clear(remove) any resolver only if it exists. - - Returns a bool: True if resolver is removed and False if not removed. - - .. warning: - This method can remove deafult resolvers as well. - - :param name: Name of the resolver. - :return: A bool (``True`` if resolver is removed, ``False`` if not found before removing). - """ - if cls.has_resolver(name): - BaseContainer._resolvers.pop(name) - return True - else: - # return False if resolver does not exist - return False - - @staticmethod - def get_cache(conf: BaseContainer) -> Dict[str, Any]: - return conf._metadata.resolver_cache - - @staticmethod - def set_cache(conf: BaseContainer, cache: Dict[str, Any]) -> None: - conf._metadata.resolver_cache = copy.deepcopy(cache) - - @staticmethod - def clear_cache(conf: BaseContainer) -> None: - OmegaConf.set_cache(conf, defaultdict(dict, {})) - - @staticmethod - def copy_cache(from_config: BaseContainer, to_config: BaseContainer) -> None: - OmegaConf.set_cache(to_config, OmegaConf.get_cache(from_config)) - - @staticmethod - def set_readonly(conf: Node, value: Optional[bool]) -> None: - # noinspection PyProtectedMember - conf._set_flag("readonly", value) - - @staticmethod - def is_readonly(conf: Node) -> Optional[bool]: - # noinspection PyProtectedMember - return conf._get_flag("readonly") - - @staticmethod - def set_struct(conf: Container, value: Optional[bool]) -> None: - # noinspection PyProtectedMember - conf._set_flag("struct", value) - - @staticmethod - def is_struct(conf: Container) -> Optional[bool]: - # noinspection PyProtectedMember - return conf._get_flag("struct") - - @staticmethod - def masked_copy(conf: DictConfig, keys: Union[str, List[str]]) -> DictConfig: - """ - Create a masked copy of of this config that contains a subset of the keys - - :param conf: DictConfig object - :param keys: keys to preserve in the copy - :return: The masked ``DictConfig`` object. - """ - from .dictconfig import DictConfig - - if not isinstance(conf, DictConfig): - raise ValueError("masked_copy is only supported for DictConfig") - - if isinstance(keys, str): - keys = [keys] - content = {key: value for key, value in conf.items_ex(resolve=False, keys=keys)} - return DictConfig(content=content) - - @staticmethod - def to_container( - cfg: Any, - *, - resolve: bool = False, - throw_on_missing: bool = False, - enum_to_str: bool = False, - structured_config_mode: SCMode = SCMode.DICT, - ) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: - """ - Recursively converts an OmegaConf config to a primitive container (dict or list). - - :param cfg: the config to convert - :param resolve: True to resolve all values - :param throw_on_missing: When True, raise MissingMandatoryValue if any missing values are present. - When False (the default), replace missing values with the string "???" in the output container. - :param enum_to_str: True to convert Enum keys and values to strings - :param structured_config_mode: Specify how Structured Configs (DictConfigs backed by a dataclass) are handled. - - By default (``structured_config_mode=SCMode.DICT``) structured configs are converted to plain dicts. - - If ``structured_config_mode=SCMode.DICT_CONFIG``, structured config nodes will remain as DictConfig. - - If ``structured_config_mode=SCMode.INSTANTIATE``, this function will instantiate structured configs - (DictConfigs backed by a dataclass), by creating an instance of the underlying dataclass. - - See also OmegaConf.to_object. - :return: A dict or a list representing this config as a primitive container. - """ - if not OmegaConf.is_config(cfg): - raise ValueError(f"Input cfg is not an OmegaConf config object ({type_str(type(cfg))})") - - return BaseContainer._to_content( - cfg, - resolve=resolve, - throw_on_missing=throw_on_missing, - enum_to_str=enum_to_str, - structured_config_mode=structured_config_mode, - ) - - @staticmethod - def to_object(cfg: Any) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: - """ - Recursively converts an OmegaConf config to a primitive container (dict or list). - Any DictConfig objects backed by dataclasses or attrs classes are instantiated - as instances of those backing classes. - - This is an alias for OmegaConf.to_container(..., resolve=True, throw_on_missing=True, - structured_config_mode=SCMode.INSTANTIATE) - - :param cfg: the config to convert - :return: A dict or a list or dataclass representing this config. - """ - return OmegaConf.to_container( - cfg=cfg, - resolve=True, - throw_on_missing=True, - enum_to_str=False, - structured_config_mode=SCMode.INSTANTIATE, - ) - - @staticmethod - def is_missing(cfg: Any, key: DictKeyType) -> bool: - assert isinstance(cfg, Container) - try: - node = cfg._get_child(key) - if node is None: - return False - assert isinstance(node, Node) - return node._is_missing() - except (UnsupportedInterpolationType, KeyError, AttributeError): - return False - - @staticmethod - def is_interpolation(node: Any, key: Optional[Union[int, str]] = None) -> bool: - if key is not None: - assert isinstance(node, Container) - target = node._get_child(key) - else: - target = node - if target is not None: - assert isinstance(target, Node) - return target._is_interpolation() - return False - - @staticmethod - def is_list(obj: Any) -> bool: - from . import ListConfig - - return isinstance(obj, ListConfig) - - @staticmethod - def is_dict(obj: Any) -> bool: - from . import DictConfig - - return isinstance(obj, DictConfig) - - @staticmethod - def is_config(obj: Any) -> bool: - from . import Container - - return isinstance(obj, Container) - - @staticmethod - def get_type(obj: Any, key: Optional[str] = None) -> Optional[Type[Any]]: - if key is not None: - c = obj._get_child(key) - else: - c = obj - return OmegaConf._get_obj_type(c) - - @staticmethod - def select( - cfg: Container, - key: str, - *, - default: Any = _DEFAULT_MARKER_, - throw_on_resolution_failure: bool = True, - throw_on_missing: bool = False, - ) -> Any: - """ - :param cfg: Config node to select from - :param key: Key to select - :param default: Default value to return if key is not found - :param throw_on_resolution_failure: Raise an exception if an interpolation - resolution error occurs, otherwise return None - :param throw_on_missing: Raise an exception if an attempt to select a missing key (with the value '???') - is made, otherwise return None - :return: selected value or None if not found. - """ - from ._impl import select_value - - try: - return select_value( - cfg=cfg, - key=key, - default=default, - throw_on_resolution_failure=throw_on_resolution_failure, - throw_on_missing=throw_on_missing, - ) - except Exception as e: - format_and_raise(node=cfg, key=key, value=None, cause=e, msg=str(e)) - - @staticmethod - def update( - cfg: Container, - key: str, - value: Any = None, - *, - merge: bool = True, - force_add: bool = False, - ) -> None: - """ - Updates a dot separated key sequence to a value - - :param cfg: input config to update - :param key: key to update (can be a dot separated path) - :param value: value to set, if value if a list or a dict it will be merged or set - depending on merge_config_values - :param merge: If value is a dict or a list, True (default) to merge - into the destination, False to replace the destination. - :param force_add: insert the entire path regardless of Struct flag or Structured Config nodes. - """ - - split = split_key(key) - root = cfg - for i in range(len(split) - 1): - k = split[i] - # if next_root is a primitive (string, int etc) replace it with an empty map - next_root, key_ = _select_one(root, k, throw_on_missing=False) - if not isinstance(next_root, Container): - if force_add: - with flag_override(root, "struct", False): - root[key_] = {} - else: - root[key_] = {} - root = root[key_] - - last = split[-1] - - assert isinstance(root, Container), f"Unexpected type for root: {type(root).__name__}" - - last_key: Union[str, int] = last - if isinstance(root, ListConfig): - last_key = int(last) - - ctx = flag_override(root, "struct", False) if force_add else nullcontext() - with ctx: - if merge and (OmegaConf.is_config(value) or is_primitive_container(value)): - assert isinstance(root, BaseContainer) - node = root._get_child(last_key) - if OmegaConf.is_config(node): - assert isinstance(node, BaseContainer) - node.merge_with(value) - return - - if OmegaConf.is_dict(root): - assert isinstance(last_key, str) - root.__setattr__(last_key, value) - elif OmegaConf.is_list(root): - assert isinstance(last_key, int) - root.__setitem__(last_key, value) - else: - assert False - - @staticmethod - def to_yaml(cfg: Any, *, resolve: bool = False, sort_keys: bool = False) -> str: - """ - returns a yaml dump of this config object. - - :param cfg: Config object, Structured Config type or instance - :param resolve: if True, will return a string with the interpolations resolved, otherwise - interpolations are preserved - :param sort_keys: If True, will print dict keys in sorted order. default False. - :return: A string containing the yaml representation. - """ - cfg = _ensure_container(cfg) - container = OmegaConf.to_container(cfg, resolve=resolve, enum_to_str=True) - return yaml.dump( # type: ignore - container, - default_flow_style=False, - allow_unicode=True, - sort_keys=sort_keys, - Dumper=get_omega_conf_dumper(), - ) - - @staticmethod - def resolve(cfg: Container) -> None: - """ - Resolves all interpolations in the given config object in-place. - - :param cfg: An OmegaConf container (DictConfig, ListConfig) - Raises a ValueError if the input object is not an OmegaConf container. - """ - import omegaconf._impl - - if not OmegaConf.is_config(cfg): - # Since this function is mutating the input object in-place, it doesn't make sense to - # auto-convert the input object to an OmegaConf container - raise ValueError(f"Invalid config type ({type(cfg).__name__}), expected an OmegaConf Container") - omegaconf._impl._resolve(cfg) - - @staticmethod - def missing_keys(cfg: Any) -> Set[str]: - """ - Returns a set of missing keys in a dotlist style. - - :param cfg: An ``OmegaConf.Container``, - or a convertible object via ``OmegaConf.create`` (dict, list, ...). - :return: set of strings of the missing keys. - :raises ValueError: On input not representing a config. - """ - cfg = _ensure_container(cfg) - missings: Set[str] = set() - - def gather(_cfg: Container) -> None: - itr: Iterable[Any] - if isinstance(_cfg, ListConfig): - itr = range(len(_cfg)) - else: - itr = _cfg - - for key in itr: - if OmegaConf.is_missing(_cfg, key): - missings.add(_cfg._get_full_key(key)) - elif OmegaConf.is_config(_cfg[key]): - gather(_cfg[key]) - - gather(cfg) - return missings - - # === private === # - - @staticmethod - def _create_impl( # noqa F811 - obj: Any = _DEFAULT_MARKER_, - parent: Optional[BaseContainer] = None, - flags: Optional[Dict[str, bool]] = None, - ) -> Union[DictConfig, ListConfig]: - try: - from ._utils import get_yaml_loader - from .dictconfig import DictConfig - from .listconfig import ListConfig - - if obj is _DEFAULT_MARKER_: - obj = {} - if isinstance(obj, str): - obj = yaml.load(obj, Loader=get_yaml_loader()) - if obj is None: - return OmegaConf.create({}, parent=parent, flags=flags) - elif isinstance(obj, str): - return OmegaConf.create({obj: None}, parent=parent, flags=flags) - else: - assert isinstance(obj, (list, dict)) - return OmegaConf.create(obj, parent=parent, flags=flags) - - else: - if ( - is_primitive_dict(obj) - or OmegaConf.is_dict(obj) - or is_structured_config(obj) - or obj is None - ): - if isinstance(obj, DictConfig): - return DictConfig( - content=obj, - parent=parent, - ref_type=obj._metadata.ref_type, - is_optional=obj._metadata.optional, - key_type=obj._metadata.key_type, - element_type=obj._metadata.element_type, - flags=flags, - ) - else: - obj_type = OmegaConf.get_type(obj) - key_type, element_type = get_dict_key_value_types(obj_type) - return DictConfig( - content=obj, - parent=parent, - key_type=key_type, - element_type=element_type, - flags=flags, - ) - elif is_primitive_list(obj) or OmegaConf.is_list(obj): - if isinstance(obj, ListConfig): - return ListConfig( - content=obj, - parent=parent, - element_type=obj._metadata.element_type, - ref_type=obj._metadata.ref_type, - is_optional=obj._metadata.optional, - flags=flags, - ) - else: - obj_type = OmegaConf.get_type(obj) - element_type = get_list_element_type(obj_type) - return ListConfig( - content=obj, - parent=parent, - element_type=element_type, - ref_type=Any, - is_optional=True, - flags=flags, - ) - else: - if isinstance(obj, type): - raise ValidationError( - f"Input class '{obj.__name__}' is not a structured config. " - "did you forget to decorate it as a dataclass?" - ) - else: - raise ValidationError(f"Object of unsupported type: '{type(obj).__name__}'") - except OmegaConfBaseException as e: - format_and_raise(node=None, key=None, value=None, msg=str(e), cause=e) - assert False - - @staticmethod - def _get_obj_type(c: Any) -> Optional[Type[Any]]: - if is_structured_config(c): - return get_type_of(c) - elif c is None: - return None - elif isinstance(c, DictConfig): - if c._is_none(): - return None - elif c._is_missing(): - return None - else: - if is_structured_config(c._metadata.object_type): - return c._metadata.object_type - else: - return dict - elif isinstance(c, ListConfig): - return list - elif isinstance(c, ValueNode): - return type(c._value()) - elif isinstance(c, UnionNode): - return type(_get_value(c)) - elif isinstance(c, dict): - return dict - elif isinstance(c, (list, tuple)): - return list - else: - return get_type_of(c) - - @staticmethod - def _get_resolver( - name: str, - ) -> Optional[Callable[[Container, Container, Node, Tuple[Any, ...], Tuple[str, ...]], Any,]]: - # noinspection PyProtectedMember - return BaseContainer._resolvers[name] if name in BaseContainer._resolvers else None - - -# register all default resolvers -register_default_resolvers() - - -@contextmanager -def flag_override( - config: Node, - names: Union[List[str], str], - values: Union[List[Optional[bool]], Optional[bool]], -) -> Generator[Node, None, None]: - if isinstance(names, str): - names = [names] - if values is None or isinstance(values, bool): - values = [values] - - prev_states = [config._get_node_flag(name) for name in names] - - try: - config._set_flag(names, values) - yield config - finally: - config._set_flag(names, prev_states) - - -@contextmanager -def read_write(config: Node) -> Generator[Node, None, None]: - prev_state = config._get_node_flag("readonly") - try: - OmegaConf.set_readonly(config, False) - yield config - finally: - OmegaConf.set_readonly(config, prev_state) - - -@contextmanager -def open_dict(config: Container) -> Generator[Container, None, None]: - prev_state = config._get_node_flag("struct") - try: - OmegaConf.set_struct(config, False) - yield config - finally: - OmegaConf.set_struct(config, prev_state) - - -# === private === # - - -def _node_wrap( - parent: Optional[Box], - is_optional: bool, - value: Any, - key: Any, - ref_type: Any = Any, -) -> Node: - node: Node - if is_dict_annotation(ref_type) or (is_primitive_dict(value) and ref_type is Any): - key_type, element_type = get_dict_key_value_types(ref_type) - node = DictConfig( - content=value, - key=key, - parent=parent, - ref_type=ref_type, - is_optional=is_optional, - key_type=key_type, - element_type=element_type, - ) - elif (is_list_annotation(ref_type) or is_tuple_annotation(ref_type)) or ( - type(value) in (list, tuple) and ref_type is Any - ): - element_type = get_list_element_type(ref_type) - node = ListConfig( - content=value, - key=key, - parent=parent, - is_optional=is_optional, - element_type=element_type, - ref_type=ref_type, - ) - elif is_structured_config(ref_type) or is_structured_config(value): - key_type, element_type = get_dict_key_value_types(value) - node = DictConfig( - ref_type=ref_type, - is_optional=is_optional, - content=value, - key=key, - parent=parent, - key_type=key_type, - element_type=element_type, - ) - elif is_union_annotation(ref_type): - node = UnionNode( - content=value, - ref_type=ref_type, - is_optional=is_optional, - key=key, - parent=parent, - ) - elif ref_type == Any or ref_type is None: - node = AnyNode(value=value, key=key, parent=parent) - elif isinstance(ref_type, type) and issubclass(ref_type, Enum): - node = EnumNode( - enum_type=ref_type, - value=value, - key=key, - parent=parent, - is_optional=is_optional, - ) - elif ref_type == int: - node = IntegerNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif ref_type == float: - node = FloatNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif ref_type == bool: - node = BooleanNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif ref_type == str: - node = StringNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif ref_type == bytes: - node = BytesNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif ref_type == pathlib.Path: - node = PathNode(value=value, key=key, parent=parent, is_optional=is_optional) - else: - if parent is not None and parent._get_flag("allow_objects") is True: - if type(value) in (list, tuple): - node = ListConfig( - content=value, - key=key, - parent=parent, - ref_type=ref_type, - is_optional=is_optional, - ) - elif is_primitive_dict(value): - node = DictConfig( - content=value, - key=key, - parent=parent, - ref_type=ref_type, - is_optional=is_optional, - ) - else: - node = AnyNode(value=value, key=key, parent=parent) - else: - raise ValidationError(f"Unexpected type annotation: {type_str(ref_type)}") - return node - - -def _maybe_wrap( - ref_type: Any, - key: Any, - value: Any, - is_optional: bool, - parent: Optional[BaseContainer], -) -> Node: - # if already a node, update key and parent and return as is. - # NOTE: that this mutate the input node! - if isinstance(value, Node): - value._set_key(key) - value._set_parent(parent) - return value - else: - return _node_wrap( - ref_type=ref_type, - parent=parent, - is_optional=is_optional, - value=value, - key=key, - ) - - -def _select_one( - c: Container, key: str, throw_on_missing: bool, throw_on_type_error: bool = True -) -> Tuple[Optional[Node], Union[str, int]]: - from .dictconfig import DictConfig - from .listconfig import ListConfig - - ret_key: Union[str, int] = key - assert isinstance(c, Container), f"Unexpected type: {c}" - if c._is_none(): - return None, ret_key - - if isinstance(c, DictConfig): - assert isinstance(ret_key, str) - val = c._get_child(ret_key, validate_access=False) - elif isinstance(c, ListConfig): - assert isinstance(ret_key, str) - if not is_int(ret_key): - if throw_on_type_error: - raise TypeError(f"Index '{ret_key}' ({type(ret_key).__name__}) is not an int") - else: - val = None - else: - ret_key = int(ret_key) - if ret_key < 0 or ret_key + 1 > len(c): - val = None - else: - val = c._get_child(ret_key) - else: - assert False - - if val is not None: - assert isinstance(val, Node) - if val._is_missing(): - if throw_on_missing: - raise MissingMandatoryValue(f"Missing mandatory value: {c._get_full_key(ret_key)}") - else: - return val, ret_key - - assert val is None or isinstance(val, Node) - return val, ret_key diff --git a/omegaconf/py.typed b/omegaconf/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/omegaconf/resolvers/__init__.py b/omegaconf/resolvers/__init__.py deleted file mode 100644 index 1b0bb7116..000000000 --- a/omegaconf/resolvers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from omegaconf.resolvers import oc - -__all__ = [ - "oc", -] diff --git a/omegaconf/resolvers/oc/__init__.py b/omegaconf/resolvers/oc/__init__.py deleted file mode 100644 index a8e0e9e66..000000000 --- a/omegaconf/resolvers/oc/__init__.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -import string -import warnings -from typing import Any, Optional - -from omegaconf import Container, Node -from omegaconf._utils import _DEFAULT_MARKER_, _get_value -from omegaconf.basecontainer import BaseContainer -from omegaconf.errors import ConfigKeyError -from omegaconf.grammar_parser import parse -from omegaconf.resolvers.oc import dict - - -def create(obj: Any, _parent_: Container) -> Any: - """Create a config object from `obj`, similar to `OmegaConf.create`""" - from omegaconf import OmegaConf - - assert isinstance(_parent_, BaseContainer) - return OmegaConf.create(obj, parent=_parent_) - - -def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]: - """ - :param key: Environment variable key - :param default: Optional default value to use in case the key environment variable is not set. - If default is not a string, it is converted with str(default). - None default is returned as is. - :return: The environment variable 'key'. If the environment variable is not set and a default is - provided, the default is used. If used, the default is converted to a string with str(default). - If the default is None, None is returned (without a string conversion). - """ - try: - return os.environ[key] - except KeyError: - if default is not _DEFAULT_MARKER_: - return str(default) if default is not None else None - else: - raise KeyError(f"Environment variable '{key}' not found") - - -def decode(expr: Optional[str], _parent_: Container, _node_: Node) -> Any: - """ - Parse and evaluate `expr` according to the `singleElement` rule of the grammar. - - If `expr` is `None`, then return `None`. - """ - if expr is None: - return None - - if not isinstance(expr, str): - raise TypeError( - f"`oc.decode` can only take strings or None as input, " - f"but `{expr}` is of type {type(expr).__name__}" - ) - - parse_tree = parse(expr, parser_rule="singleElement", lexer_mode="VALUE_MODE") - val = _parent_.resolve_parse_tree(parse_tree, node=_node_) - return _get_value(val) - - -def deprecated( - key: str, - message: str = "'$OLD_KEY' is deprecated. Change your code and config to use '$NEW_KEY'", - *, - _parent_: Container, - _node_: Node, -) -> Any: - from omegaconf._impl import select_node - - if not isinstance(key, str): - raise TypeError(f"oc.deprecated: interpolation key type is not a string ({type(key).__name__})") - - if not isinstance(message, str): - raise TypeError( - f"oc.deprecated: interpolation message type is not a string ({type(message).__name__})" - ) - - full_key = _node_._get_full_key(key=None) - target_node = select_node(_parent_, key, absolute_key=True) - if target_node is None: - raise ConfigKeyError(f"In oc.deprecated resolver at '{full_key}': Key not found: '{key}'") - new_key = target_node._get_full_key(key=None) - msg = string.Template(message).safe_substitute( - OLD_KEY=full_key, - NEW_KEY=new_key, - ) - warnings.warn(category=UserWarning, message=msg) - return target_node - - -def select( - key: str, - default: Any = _DEFAULT_MARKER_, - *, - _parent_: Container, -) -> Any: - from omegaconf._impl import select_value - - return select_value(cfg=_parent_, key=key, absolute_key=True, default=default) - - -__all__ = [ - "create", - "decode", - "deprecated", - "dict", - "env", - "select", -] diff --git a/omegaconf/resolvers/oc/dict.py b/omegaconf/resolvers/oc/dict.py deleted file mode 100644 index 3bcb06b8f..000000000 --- a/omegaconf/resolvers/oc/dict.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import Any, List - -from omegaconf import AnyNode, Container, DictConfig, ListConfig -from omegaconf._utils import Marker -from omegaconf.basecontainer import BaseContainer -from omegaconf.errors import ConfigKeyError - -_DEFAULT_SELECT_MARKER_: Any = Marker("_DEFAULT_SELECT_MARKER_") - - -def keys( - key: str, - _parent_: Container, -) -> ListConfig: - from omegaconf import OmegaConf - - assert isinstance(_parent_, BaseContainer) - - in_dict = _get_and_validate_dict_input(key, parent=_parent_, resolver_name="oc.dict.keys") - - ret = OmegaConf.create(list(in_dict.keys()), parent=_parent_) - assert isinstance(ret, ListConfig) - return ret - - -def values(key: str, _root_: BaseContainer, _parent_: Container) -> ListConfig: - assert isinstance(_parent_, BaseContainer) - in_dict = _get_and_validate_dict_input(key, parent=_parent_, resolver_name="oc.dict.values") - - content = in_dict._content - assert isinstance(content, dict) - - ret = ListConfig([]) - if key.startswith("."): - key = f".{key}" # extra dot to compensate for extra level of nesting within ret ListConfig - for k in content: - ref_node = AnyNode(f"${{{key}.{k!s}}}") - ret.append(ref_node) - - # Finalize result by setting proper type and parent. - element_type: Any = in_dict._metadata.element_type - ret._metadata.element_type = element_type - ret._metadata.ref_type = List[element_type] - ret._set_parent(_parent_) - - return ret - - -def _get_and_validate_dict_input( - key: str, - parent: BaseContainer, - resolver_name: str, -) -> DictConfig: - from omegaconf._impl import select_value - - if not isinstance(key, str): - raise TypeError( - f"`{resolver_name}` requires a string as input, but obtained `{key}` " - f"of type: {type(key).__name__}" - ) - - in_dict = select_value( - parent, - key, - throw_on_missing=True, - absolute_key=True, - default=_DEFAULT_SELECT_MARKER_, - ) - - if in_dict is _DEFAULT_SELECT_MARKER_: - raise ConfigKeyError(f"Key not found: '{key}'") - - if not isinstance(in_dict, DictConfig): - raise TypeError( - f"`{resolver_name}` cannot be applied to objects of type: " f"{type(in_dict).__name__}" - ) - - return in_dict diff --git a/omegaconf/version.py b/omegaconf/version.py deleted file mode 100644 index f73c6b765..000000000 --- a/omegaconf/version.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys # pragma: no cover - -__version__ = "2.4.0.dev0" - -msg = """OmegaConf 2.0 and above is compatible with Python 3.6 and newer. -You have the following options: -1. Upgrade to Python 3.6 or newer. - This is highly recommended. new features will not be added to OmegaConf 1.4. -2. Continue using OmegaConf 1.4: - You can pip install 'OmegaConf<1.5' to do that. -""" -if sys.version_info < (3, 6): - raise ImportError(msg) # pragma: no cover From 91a85e8e8f54eedee6b3f9d4f00dcf912e738ef0 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:51:54 +0200 Subject: [PATCH 16/37] fix: change module import path --- kapitan/omegaconf_inv.py | 2 +- kapitan/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 67cc9ba5b..69b039832 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -8,7 +8,7 @@ import time from kapitan.errors import InventoryError -from kapitan.oc.omegaconf import Node, OmegaConf, errors +from oc.omegaconf import Node, OmegaConf, errors logger = logging.getLogger(__name__) diff --git a/kapitan/resources.py b/kapitan/resources.py index 22f9b71c9..fe24b3f33 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -28,7 +28,7 @@ from kapitan.migrate_omegaconf import migrate from kapitan.omegaconf_inv import inventory_omegaconf from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -from kapitan.oc.omegaconf import errors +from oc.omegaconf import errors logger = logging.getLogger(__name__) From 3e94da2b5931baa3195a9bb904ce927e517ba52d Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:14:43 +0200 Subject: [PATCH 17/37] fix: change import paths for omegaconf --- kapitan/omegaconf_inv.py | 2 +- kapitan/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 69b039832..ebafd09ef 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -8,7 +8,7 @@ import time from kapitan.errors import InventoryError -from oc.omegaconf import Node, OmegaConf, errors +from omegaconf import Node, OmegaConf, errors logger = logging.getLogger(__name__) diff --git a/kapitan/resources.py b/kapitan/resources.py index fe24b3f33..7b65c36e2 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -28,7 +28,7 @@ from kapitan.migrate_omegaconf import migrate from kapitan.omegaconf_inv import inventory_omegaconf from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -from oc.omegaconf import errors +from omegaconf import errors logger = logging.getLogger(__name__) From 88175c695c9776d4651cfe651d5648aacb661ad2 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:31:21 +0200 Subject: [PATCH 18/37] feat: resolve relative class name --- kapitan/omegaconf_inv.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index ebafd09ef..b9344627e 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -110,12 +110,20 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # load classes for targets for class_name in target_config_classes: - # resolve class name (relative paths TBD) + # resolve class name class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" + if os.path.isfile(class_path): # load classes recursively class_config = OmegaConf.load(class_path) - target_config_classes.extend(class_config.pop("classes", [])) + + # TBD: deep relative paths (only one layer supported yet) + new_classes = class_config.pop("classes", []) + for new in new_classes: + if new.startswith("."): + new = ".".join(class_name.split(".")[0:-1]) + new + + target_config_classes.append(new) elif not ignore_class_notfound: raise InventoryError(f"{target_name}: Class {class_name} not found.") From 6b53e028493a062c6571d0f16471156f6612ac48 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 14 Jun 2023 20:10:20 +0200 Subject: [PATCH 19/37] refactor: adapt new merge interface --- kapitan/omegaconf_inv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index b9344627e..3a7bc6fce 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -132,7 +132,7 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # merge target with loaded classes if target_config.get("parameters"): - target_config = OmegaConf.merge(class_config, target_config, extend_lists=True) + target_config = OmegaConf.merge(class_config, target_config, list_merge_mode="EXTEND") else: target_config = class_config From 6147f0dd56d6b54165320cbbf280eab3f1d92366 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:28:58 +0200 Subject: [PATCH 20/37] refactor: remove examples inventory for omegaconf --- .../classes/cluster/common.yml | 3 - .../classes/cluster/minikube.yml | 19 ------ .../inventory_omegaconf/classes/common.yml | 18 ------ .../classes/component/busybox.yml | 19 ------ .../classes/component/elasticsearch.yml | 46 -------------- .../classes/component/labels.yml | 16 ----- .../classes/component/mysql.yml | 61 ------------------- .../classes/component/namespace.yml | 9 --- .../classes/component/nginx-common.yml | 12 ---- .../classes/component/nginx-helm.yml | 24 -------- .../classes/component/nginx-jsonnet.yml | 12 ---- .../classes/component/nginx-kadet.yml | 12 ---- .../inventory_omegaconf/targets/all-glob.yml | 60 ------------------ .../inventory_omegaconf/targets/busybox.yml | 6 -- .../inventory_omegaconf/targets/labels.yml | 6 -- .../targets/minikube-es.yml | 11 ---- .../targets/minikube-mysql.yml | 9 --- .../targets/minikube-nginx-helm.yml | 9 --- .../targets/minikube-nginx-jsonnet.yml | 10 --- .../targets/minikube-nginx-kadet.yml | 11 ---- .../inventory_omegaconf/targets/removal.yml | 14 ----- 21 files changed, 387 deletions(-) delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/common.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/labels.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/all-glob.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/busybox.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/labels.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml delete mode 100644 examples/kubernetes/inventory_omegaconf/targets/removal.yml diff --git a/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml b/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml deleted file mode 100644 index 36ed23592..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/cluster/common.yml +++ /dev/null @@ -1,3 +0,0 @@ -parameters: - kubectl: - insecure_skip_tls_verify: false diff --git a/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml b/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml deleted file mode 100644 index b8ee0ebc5..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/cluster/minikube.yml +++ /dev/null @@ -1,19 +0,0 @@ -# class for all targets deployed on minikube -# -classes: - - cluster.common -parameters: - minikube: - memory: 4096 - cpus: 4 - version: v0.31.0 - - cluster: - type: minikube - id: minikube - name: minikube - user: minikube - vault: - address: https://localhost:8200 - mysql: - hostname: localhost diff --git a/examples/kubernetes/inventory_omegaconf/classes/common.yml b/examples/kubernetes/inventory_omegaconf/classes/common.yml deleted file mode 100644 index 06e36f4d0..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/common.yml +++ /dev/null @@ -1,18 +0,0 @@ -parameters: - namespace: ${parameters.target_name} - target_name: ${parameters._reclass_.name.short} - - kapitan: - vars: - target: ${parameters.target_name} - namespace: ${parameters.target_name} - managed_by: kapitan - secrets: - gpg: - recipients: - - name: example@kapitan.dev - fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C - gkms: - key: 'projects//locations//keyRings//cryptoKeys/' - awskms: - key: 'alias/nameOfKey' diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml b/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml deleted file mode 100644 index f664cde51..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/busybox.yml +++ /dev/null @@ -1,19 +0,0 @@ -parameters: - kapitan: - vars: - target: ${parameters.target_name} - namespace: ${parameters.target_name} - compile: - - output_path: ./copy - input_type: copy - input_paths: - - components/busybox/pod.yml - - input_type: copy - input_paths: - - copy_target - output_path: ./copy - # test copying over root output_path - - input_type: copy - input_paths: - - copy_target - output_path: . diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml b/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml deleted file mode 100644 index 4d88ad628..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/elasticsearch.yml +++ /dev/null @@ -1,46 +0,0 @@ -parameters: - elasticsearch: - image: "quay.io/pires/docker-elasticsearch-kubernetes:5.5.0" - java_opts: "-Xms512m -Xmx512m" - replicas: 1 - masters: 1 - roles: - master: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - data: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - client: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - ingest: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - - kapitan: - vars: - target: ${parameters.target_name} - namespace: ${parameters.target_name} - compile: - - output_path: manifests - input_type: jsonnet - input_paths: - - components/elasticsearch/main.jsonnet - output_type: yml - - output_path: scripts - input_type: jinja2 - input_paths: - - scripts - - output_path: . - input_type: jinja2 - input_paths: - - docs/elasticsearch/README.md diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml b/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml deleted file mode 100644 index 398be2969..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/labels.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - postgres: - name: db - component: db - instance: postgres - version: "11.0" - - kapitan: - vars: - target: ${parameters.target_name} - namespace: ${parameters.target_name} - compile: - - output_path: ./labels - input_type: jinja2 - input_paths: - - components/labels/service.yml diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml b/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml deleted file mode 100644 index b0f24a2ab..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/mysql.yml +++ /dev/null @@ -1,61 +0,0 @@ -parameters: - kapitan: - compile: - - output_path: manifests - input_type: jsonnet - prune: true - input_paths: - - components/mysql/main.jsonnet - output_type: yml - - output_path: scripts - input_type: jinja2 - input_paths: - - scripts - - output_path: . - output_type: yml - input_type: jinja2 - input_paths: - - docs/mysql/README.md - validate: - - type: kubernetes # mkdocs (1)! - output_paths: # mkdocs (2)! - - manifests/mysql_secret.yml - kind: secret # temporarily replaced with 'deployment' during test # mkdocs (3)! - version: 1.14.0 # optional, defaults to 1.14.0 # mkdocs (4)! - - type: kubernetes - output_paths: - - manifests/mysql_service_jsonnet.yml - - manifests/mysql_service_simple.yml - kind: service - version: 1.14.0 - # For vaultkv secrets it is important to declare auth in parameters - secrets: - vaultkv: - auth: token - vaulttransit: - auth: token - key: my-vault-key - mysql: - storage: 10G - storage_class: standard - image: mysql:latest - users: - root: - # If 'secrets/targets/${target_name}/mysql/password' doesn't exist, it will gen a random b64-encoded password - password: ?{gpg:targets/${parameters.target_name}/mysql/password||randomstr|base64} - # password: ?{gkms:targets/${target_name}/mysql/password||randomstr|base64} - # password: ?{awskms:targets/${target_name}/mysql/password||randomstr|base64} - - # Generates the sha256 checksum of the previously declared B64'ed password - # It's base64'ed again so that it can be used in kubernetes secrets - password_sha256: - ?{gpg:targets/${parameters.target_name}/mysql/password_sha256||reveal:targets/${parameters.target_name}/mysql/password|sha256|base64} - - password_subvar: ?{gpg:targets/${parameters.target_name}/mysql/subvars@var1.password} - password_sha256_subvar: ?{gpg:targets/${parameters.target_name}/mysql/subvars@var2.password_sha256} - - # This secret requires a running vault instance & vaultkv parameters either in inventory or environment - # password_vaultkv: ?{vaultkv:targets/${target_name}/mysql/vault_secret} - - # This secret requires a running vault instance & vaulttransit parameters either in inventory or environment - # password_transit: ?{transit:${target_name}/mysql/vault_secret} diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml b/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml deleted file mode 100644 index ba82d74ac..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/namespace.yml +++ /dev/null @@ -1,9 +0,0 @@ -parameters: - namespace: ${parameters.target_name} - kapitan: - compile: - - output_path: pre-deploy - input_type: jsonnet - output_type: yml - input_paths: - - components/namespace/main.jsonnet diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml deleted file mode 100644 index f4d62f1a3..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-common.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: - nginx: - image: nginx:1:15.8 - - templates: #(1)! - - docs/nginx/README.md - - components/nginx-deploy.sh - kapitan: - compile: - - output_path: . #(2)! - input_type: jinja2 - input_paths: ${parameters.templates} #(3)! diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml deleted file mode 100644 index 18b944e3a..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-helm.yml +++ /dev/null @@ -1,24 +0,0 @@ -parameters: - namespace: - nginx: - version: 4.4.0 - replicas: 2 - name: ${parameters.target_name} - namespace: ${parameters.target_name} - kapitan: - dependencies: - - type: helm - output_path: charts/nginx-ingress - source: https://kubernetes.github.io/ingress-nginx - chart_name: ingress-nginx - compile: - - output_path: . - input_type: helm - input_paths: - - charts/nginx-ingress - helm_values: - controller: - name: ${parameters.nginx.name} - helm_params: - name: ${parameters.nginx.name} - namespace: ${parameters.nginx.namespace} diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml deleted file mode 100644 index 773c33e68..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-jsonnet.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: - nginx: - replicas: 1 - kapitan: - compile: - - output_path: manifests - output_type: yml - input_type: jsonnet - input_paths: - - components/nginx-jsonnet/main.jsonnet - labels: - type: jsonnet diff --git a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml b/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml deleted file mode 100644 index ff9e2f919..000000000 --- a/examples/kubernetes/inventory_omegaconf/classes/component/nginx-kadet.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: - nginx: - replicas: 2 - kapitan: - compile: - - output_path: manifests - input_type: kadet - output_type: yml - input_paths: - - components/nginx-kadet/ - labels: - type: kadet diff --git a/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml b/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml deleted file mode 100644 index 319c852b7..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/all-glob.yml +++ /dev/null @@ -1,60 +0,0 @@ -classes: - - common - - component.namespace - - cluster.minikube -parameters: - target_name: all-glob - - elasticsearch: - image: "quay.io/pires/docker-elasticsearch-kubernetes:5.5.0" - java_opts: "-Xms512m -Xmx512m" - replicas: 1 - masters: 1 - roles: - ingest: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - master: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - data: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - client: - image: ${parameters.elasticsearch.image} - java_opts: ${parameters.elasticsearch.java_opts} - replicas: ${parameters.elasticsearch.replicas} - masters: ${parameters.elasticsearch.masters} - - mysql: - instance_name: glob_instance - image: mysql:latest - storage_class: standard - storage: 10G - users: - root: - password: ?{base64:targets/${parameters.target_name}/mysql/password||randomstr|base64} - password_sha256: - ?{base64:targets/${parameters.target_name}/mysql/password_sha256||reveal:targets/${parameters.target_name}/mysql/password|sha256|base64} - password_subvar: ?{base64:targets/${parameters.target_name}/mysql/subvars@var1.password} - password_sha256_subvar: ?{base64:targets/${parameters.target_name}/mysql/subvars@var2.password_sha256} - - nginx: - image: nginx:1:15.8 - replicas: 2 - - # compile all main.jsonnet in components/ - kapitan: - compile: - - output_path: manifests - input_type: jsonnet - input_paths: - - components/*/main.jsonnet - output_type: yml - diff --git a/examples/kubernetes/inventory_omegaconf/targets/busybox.yml b/examples/kubernetes/inventory_omegaconf/targets/busybox.yml deleted file mode 100644 index 16622e715..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/busybox.yml +++ /dev/null @@ -1,6 +0,0 @@ -classes: - - common - - component.namespace - - component.busybox -parameters: - target_name: busybox diff --git a/examples/kubernetes/inventory_omegaconf/targets/labels.yml b/examples/kubernetes/inventory_omegaconf/targets/labels.yml deleted file mode 100644 index 2215b5349..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/labels.yml +++ /dev/null @@ -1,6 +0,0 @@ -classes: - - common - - component.namespace - - component.labels -parameters: - target_name: labels diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml deleted file mode 100644 index c33985517..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/minikube-es.yml +++ /dev/null @@ -1,11 +0,0 @@ -classes: - - common - - cluster.minikube - - component.namespace - - component.elasticsearch - - component.busybox -parameters: - target_name: minikube-es - - elasticsearch: - replicas: 2 diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml deleted file mode 100644 index 1d32880c6..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/minikube-mysql.yml +++ /dev/null @@ -1,9 +0,0 @@ -classes: - - common - - cluster.minikube - - component.namespace - - component.mysql -parameters: - mysql: - instance_name: example-mysql - replicas: 1 diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml deleted file mode 100644 index f9f8162e8..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-helm.yml +++ /dev/null @@ -1,9 +0,0 @@ -classes: - - common - - cluster.minikube - - component.nginx-helm - - component.nginx-common -parameters: -# These parameters are redundand because automatically set by `common` -## target_name: minikube-nginx-helm -## namespace: ${target_name} diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml deleted file mode 100644 index 439a62f28..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-jsonnet.yml +++ /dev/null @@ -1,10 +0,0 @@ -classes: - - common - - cluster.minikube - - component.namespace - - component.nginx-jsonnet - - component.nginx-common -parameters: -# These parameters are redundand because automatically set by `common` -## target_name: minikube-nginx-jsonnet -## namespace: ${target_name} diff --git a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml b/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml deleted file mode 100644 index 09540fc5e..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/minikube-nginx-kadet.yml +++ /dev/null @@ -1,11 +0,0 @@ -classes: - - common - - cluster.minikube - - component.namespace - - component.nginx-kadet - - component.nginx-common -parameters: -# These parameters are redundand because automatically set by `common` -## target_name: minikube-nginx-kadet -## namespace: ${target_name} - diff --git a/examples/kubernetes/inventory_omegaconf/targets/removal.yml b/examples/kubernetes/inventory_omegaconf/targets/removal.yml deleted file mode 100644 index c42ba1b2e..000000000 --- a/examples/kubernetes/inventory_omegaconf/targets/removal.yml +++ /dev/null @@ -1,14 +0,0 @@ -classes: - - common -parameters: - kapitan: - compile: - - input_type: copy - input_paths: - - copy_target - output_path: . - # test removal of a file - - input_type: remove - input_paths: - - compiled/${parameters.kapitan.vars.target}/copy_target - output_path: . From f38696f33ebb07ccbe5dba6af069aae22e04b258 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:29:59 +0200 Subject: [PATCH 21/37] feat: change migration via flag, not query --- kapitan/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 5a4beb945..63e748518 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -310,7 +310,14 @@ def build_parser(): "--omegaconf", help="use omegaconf as inventory backend", action="store_true", - default=from_dot_kapitan("inventory", "omegaconf", False), + default=from_dot_kapitan("compile", "omegaconf", False), + ) + + compile_parser.add_argument( + "--migrate", + help="migrate inventory to omegaconf", + action="store_true", + default=from_dot_kapitan("compile", "migrate", False), ) compile_selector_parser = compile_parser.add_mutually_exclusive_group() From 45d44a3c26f3e0e15b28901d340449943b0ad0b5 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:30:29 +0200 Subject: [PATCH 22/37] refactor: remove unneccessary debug and comments --- kapitan/omegaconf_inv.py | 112 ++++++++++++++++++--------------------- kapitan/resources.py | 25 ++++----- 2 files changed, 63 insertions(+), 74 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 3a7bc6fce..6435e4c34 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -2,42 +2,30 @@ # Copyright 2023 nexenio import logging -from multiprocessing.pool import ThreadPool -from functools import partial import os -import time from kapitan.errors import InventoryError -from omegaconf import Node, OmegaConf, errors +from omegaconf import Node, OmegaConf, errors, ListMergeMode logger = logging.getLogger(__name__) -def inventory_omegaconf(inventory_path, ignore_class_notfound=False, targets=[], compose_node_name=False): +def inventory_omegaconf( + inventory_path: str, + ignore_class_notfound: bool = False, + targets: list = [], + compose_node_name: bool = False, +) -> dict: """ generates inventory from yaml files using OmegaConf """ - # TBD: add config to specify paths (do we need that?) + + # add config option to specify paths targets_searchpath = os.path.join(inventory_path, "targets") classes_searchpath = os.path.join(inventory_path, "classes") - def key(_node_: Node): - """resolver function, that returns the name of its parent key""" - return _node_._key() - - def fullkey(_node_: Node): - """resolver function, that returns the full name of its parent key""" - return _node_._get_full_key("") - - def dep(name: str): - """resolver function, that returns a parameterized dependency""" - return f"dependency{name}" - - OmegaConf.register_new_resolver("key", key) - OmegaConf.register_new_resolver("fullkey", fullkey) + register_resolvers() - # kapitan helpers - OmegaConf.register_new_resolver("dep", dep) selected_targets = [] # loop through targets searchpath and load all targets @@ -64,60 +52,40 @@ def dep(name: str): selected_targets.append({"name": target_name, "path": target_path}) inv = {"nodes": {}} - # load targets parallel - stamp = time.time() + # load targets for target in selected_targets: try: name, config = load_target(target, classes_searchpath, ignore_class_notfound) inv["nodes"][name] = config except Exception as e: - print(target, e) + logger.error(f"{target['name']}: {e}") - # pool = ThreadPool(8) - - # worker = partial( - # load_target, - # classes_searchpath=classes_searchpath, - # ignore_class_notfound=ignore_class_notfound, - # ) - - # for p in pool.imap(worker, selected_targets): - # name, config = p - # inv["nodes"][name] = config - - # pool.close() - - print(f"real time: {time.time() - stamp}") - - # TBD: refactor inventory accessing (targets.py, cmd_parser.py) - # that it only receives the targets and not everything return inv def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bool = False): - # load the target + """ + load only one target with all its classes + """ + target_name = target["name"] target_path = target["path"] - try: - target_config = OmegaConf.load(target_path) - except: - print(target) - return "", {} + target_config = OmegaConf.load(target_path) target_config_classes = target_config.pop("classes", []) # load classes for targets for class_name in target_config_classes: - # resolve class name + # resolve class paths class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" if os.path.isfile(class_path): # load classes recursively class_config = OmegaConf.load(class_path) - # TBD: deep relative paths (only one layer supported yet) + # resolve relative class names new_classes = class_config.pop("classes", []) for new in new_classes: if new.startswith("."): @@ -132,7 +100,7 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # merge target with loaded classes if target_config.get("parameters"): - target_config = OmegaConf.merge(class_config, target_config, list_merge_mode="EXTEND") + target_config = OmegaConf.merge(class_config, target_config, list_merge_mode=ListMergeMode.EXTEND) else: target_config = class_config @@ -142,24 +110,50 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo if not target_config.get("parameters"): raise InventoryError(f"{target_name}: target has no parameters") - # append meta data _reclass_ (legacy) (refactoring TBD) + # append meta data _reclass_ (legacy) target_config["parameters"]["_reclass_"] = { - "name": {"full": target_name, "path": target_name, "short": target_name} + "name": { + "full": target_name, + "parts": target_name.split("."), + "path": target_name, + "short": target_name, + } } # resolve references / interpolate values try: target_config = OmegaConf.to_container(target_config, resolve=True) except errors.OmegaConfBaseException as e: - raise InventoryError(f"{target_name}: {e.__context__}") + raise InventoryError(e.__context__) - # obtain target name to insert in inv dict (legacy) (refactoring TBD) + # obtain target name to insert in inv dict try: target_name = target_config["parameters"]["kapitan"]["vars"]["target"] except KeyError: logger.warning(f"Could not resolve target name on target {target_name}") - # # stamp = time.time() - # logger.info("loaded %s in %.4f", target_name, 1000*(time.time() - stamp)) - return target_name, target_config + + +def key(_node_: Node): + """resolver function, that returns the name of its parent key""" + return _node_._key() + + +def fullkey(_node_: Node): + """resolver function, that returns the full name of its parent key""" + return _node_._get_full_key("") + + +def dep(name: str): + """resolver function, that returns a parameterized dependency""" + return f"dependency{name}" + + +def register_resolvers(): + # utils + OmegaConf.register_new_resolver("key", key) + OmegaConf.register_new_resolver("fullkey", fullkey) + + # kapitan helpers + OmegaConf.register_new_resolver("dep", dep) diff --git a/kapitan/resources.py b/kapitan/resources.py index 7b65c36e2..c6bbc320f 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -320,29 +320,24 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): # get parsed args from cached.py args = list(cached.args.values())[0] use_omegaconf = args.omegaconf + migrate_omegaconf = args.migrate if use_omegaconf: + # show warning logger.debug("Using omegaconf as inventory backend") logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") # migrate a reclass inventory to omegaConf - while True: - query_result = input(f"Do you want to migrate your inventory to OmegaConf? (Y/n):").lower() - if not query_result or query_result[0] == "y": - output_path = inventory_path # + "_omegaconf" - if not os.path.exists(output_path): - os.mkdir(output_path) - migrate(inventory_path, output_path) - logger.info(f"Migrated inventory to OmegaConf in {output_path}") - break - elif query_result[0] == "n": - break - else: - logger.warning("Please answer with 'yes'(y) or 'no'(n)!") + if migrate_omegaconf: + output_path = inventory_path + if not os.path.exists(output_path): + os.mkdir(output_path) + migrate(inventory_path, output_path) + logger.info(f"Migrated inventory to OmegaConf in {output_path}") try: inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) - except errors.OmegaConfBaseException: - raise InventoryError("") + except errors.OmegaConfBaseException as e: + raise InventoryError(e) else: logger.debug("Using reclass as inventory backend") inv = inventory_reclass(inventory_path, ignore_class_notfound) From c68437667c7d738694e5dcc248b6e83dc5bbd639 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:30:59 +0200 Subject: [PATCH 23/37] feat: add option to pull omegaconf locally --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 7434d33ea..5ed081621 100644 --- a/Makefile +++ b/Makefile @@ -57,3 +57,10 @@ local_serve_documentation: mkdocs_gh_deploy: # to run locally assuming git ssh access docker build -f Dockerfile.docs --no-cache -t kapitan-docs . docker run --rm -it -v $(PWD):/src -v ~/.ssh:/root/.ssh -w /src kapitan-docs gh-deploy -f ./mkdocs.yml + +pull_oc: + rm -rf oc + rm -rf omegaconf + git clone --branch 1080-add-list-deep-merging https://github.com/nexenio/omegaconf.git oc + pip install -r oc/requirements/dev.txt -e oc/ + mv oc/omegaconf . From 4a72a4007f0d80b698904c0f0425aa1d6c6bfbcd Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:37:10 +0200 Subject: [PATCH 24/37] lint: fix type annotations --- kapitan/migrate_omegaconf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py index 04f157469..1c767039a 100644 --- a/kapitan/migrate_omegaconf.py +++ b/kapitan/migrate_omegaconf.py @@ -4,6 +4,7 @@ import os from ruamel.yaml import YAML from pathlib import Path +from typing import Any import regex as re REF_TOKEN = r"(? str: # replace all references with OmegaConf syntax -def migrate_yaml_obj(yaml_obj: dict | list | str) -> None: +def migrate_yaml_obj(yaml_obj: Any) -> Any: # dictionary if isinstance(yaml_obj, dict): for k, v in yaml_obj.items(): From 22fb4774df513e67915235c868c16ed7e569f05f Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:38:01 +0200 Subject: [PATCH 25/37] refactor: remove directory oc after pulling --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5ed081621..ded989a72 100644 --- a/Makefile +++ b/Makefile @@ -59,8 +59,8 @@ mkdocs_gh_deploy: # to run locally assuming git ssh access docker run --rm -it -v $(PWD):/src -v ~/.ssh:/root/.ssh -w /src kapitan-docs gh-deploy -f ./mkdocs.yml pull_oc: - rm -rf oc rm -rf omegaconf git clone --branch 1080-add-list-deep-merging https://github.com/nexenio/omegaconf.git oc pip install -r oc/requirements/dev.txt -e oc/ mv oc/omegaconf . + rm -rf oc From cfc85b76b90732e4970144ea008e6753ea9d29dd Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:45:50 +0200 Subject: [PATCH 26/37] feat: support init class --- kapitan/omegaconf_inv.py | 42 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 6435e4c34..4fd4472fb 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -79,24 +79,30 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # load classes for targets for class_name in target_config_classes: # resolve class paths - class_path = os.path.join(classes_searchpath, *class_name.split(".")) + ".yml" + class_path = os.path.join(classes_searchpath, *class_name.split(".")) - if os.path.isfile(class_path): - # load classes recursively - class_config = OmegaConf.load(class_path) + # search for init file + if os.path.isdir(class_path): + init_path = os.path.join(classes_searchpath, *class_name.split("."), "init") + ".yml" + if os.path.isfile(init_path): + class_path = init_path + else: + class_path += ".yml" - # resolve relative class names - new_classes = class_config.pop("classes", []) - for new in new_classes: - if new.startswith("."): - new = ".".join(class_name.split(".")[0:-1]) + new + if not os.path.isfile(class_path): + if not ignore_class_notfound: + raise InventoryError(f"Class {class_name} not found.") - target_config_classes.append(new) + # load classes recursively + class_config = OmegaConf.load(class_path) - elif not ignore_class_notfound: - raise InventoryError(f"{target_name}: Class {class_name} not found.") - else: - continue + # resolve relative class names + new_classes = class_config.pop("classes", []) + for new in new_classes: + if new.startswith("."): + new = ".".join(class_name.split(".")[0:-1]) + new + + target_config_classes.append(new) # merge target with loaded classes if target_config.get("parameters"): @@ -105,10 +111,10 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo target_config = class_config if not target_config: - raise InventoryError(f"{target_name}: empty target") + raise InventoryError("empty target") if not target_config.get("parameters"): - raise InventoryError(f"{target_name}: target has no parameters") + raise InventoryError("target has no parameters") # append meta data _reclass_ (legacy) target_config["parameters"]["_reclass_"] = { @@ -145,9 +151,9 @@ def fullkey(_node_: Node): return _node_._get_full_key("") -def dep(name: str): +def dep(name: str, name2): """resolver function, that returns a parameterized dependency""" - return f"dependency{name}" + return {name: "abc", name2: "ghf"} def register_resolvers(): From 5155f0e5ae4ba27fe05848cee6f033e50b1a712f Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:48:06 +0200 Subject: [PATCH 27/37] perf: use faster function `unsafe_merge` --- kapitan/omegaconf_inv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 4fd4472fb..b5f079635 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -106,7 +106,7 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # merge target with loaded classes if target_config.get("parameters"): - target_config = OmegaConf.merge(class_config, target_config, list_merge_mode=ListMergeMode.EXTEND) + target_config = OmegaConf.unsafe_merge(class_config, target_config, list_merge_mode=ListMergeMode.EXTEND) else: target_config = class_config From be32b38ad5908c3310c10dfe4888dffdd46627bb Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:19:15 +0200 Subject: [PATCH 28/37] feat: add more custom resolvers --- kapitan/omegaconf_inv.py | 72 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index b5f079635..4490d605c 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -106,7 +106,9 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # merge target with loaded classes if target_config.get("parameters"): - target_config = OmegaConf.unsafe_merge(class_config, target_config, list_merge_mode=ListMergeMode.EXTEND) + target_config = OmegaConf.unsafe_merge( + class_config, target_config, list_merge_mode=ListMergeMode.EXTEND + ) else: target_config = class_config @@ -128,7 +130,7 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # resolve references / interpolate values try: - target_config = OmegaConf.to_container(target_config, resolve=True) + target_config = OmegaConf.to_object(OmegaConf.create(OmegaConf.to_object(target_config))) except errors.OmegaConfBaseException as e: raise InventoryError(e.__context__) @@ -146,20 +148,80 @@ def key(_node_: Node): return _node_._key() +def parentkey(_parent_: Node): + """resolver function, that returns the name of its parent key""" + return _parent_._key() + + def fullkey(_node_: Node): """resolver function, that returns the full name of its parent key""" return _node_._get_full_key("") -def dep(name: str, name2): +def name(name: str): + """resolver function, that returns a parameterized dependency""" + return {"name": name} + + +def merge(*args): """resolver function, that returns a parameterized dependency""" - return {name: "abc", name2: "ghf"} + merge = OmegaConf.merge(*args) + print(merge) + return merge + + +def add(o1, o2): + return o1 + o2 + + +def namespace(component: str): + return "${oc.select:parameters.components." + component + ".namespace}" + + +def deployment(component: str): + cfg = { + "namespace": f"${{oc.select:parameters.{component}-config.namespace,{component}}}", + "image": f"${{oc.select:parameters.{component}-config.image,${{parameters.config.container.base_image}}}}", + } + + cfg |= service(component) + return cfg + + +def service(component: str): + + source = { + "service": {"type": "", "selector": {"app": ""}}, + "ports": {"http": {"service_port": "187"}}, + } + + def replace(input_dict, path=""): + processed_dict = {} + + for key, value in input_dict.items(): + key_path = f"{path}.{key}" if path else key + if isinstance(value, dict): + processed_dict[key] = replace(value, path=key_path) + else: + search = f"parameters.{component}-config.{key_path}" + default = f"${{parameters.templates.deployment.{key_path}}}" if not value else value + processed_dict[key] = f"${{oc.select:{search},{default}}}" + + return processed_dict + + return replace(source) def register_resolvers(): # utils OmegaConf.register_new_resolver("key", key) + OmegaConf.register_new_resolver("parentkey", parentkey) OmegaConf.register_new_resolver("fullkey", fullkey) # kapitan helpers - OmegaConf.register_new_resolver("dep", dep) + OmegaConf.register_new_resolver("name", name) + OmegaConf.register_new_resolver("merge", merge) + OmegaConf.register_new_resolver("add", add) + OmegaConf.register_new_resolver("namespace", namespace) + OmegaConf.register_new_resolver("deployment", deployment) + OmegaConf.register_new_resolver("service", service) From 641dc73e32e05636b2827f150148a3049ca6f198 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:29:35 +0200 Subject: [PATCH 29/37] refactor: add more resolvers --- kapitan/omegaconf_inv.py | 108 ++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 40 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 4490d605c..f413a1120 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -3,6 +3,7 @@ # Copyright 2023 nexenio import logging import os +import sys from kapitan.errors import InventoryError from omegaconf import Node, OmegaConf, errors, ListMergeMode @@ -26,6 +27,14 @@ def inventory_omegaconf( register_resolvers() + # user resolvers + # import_path = os.path.join(os.getcwd(), inventory_path) + # print(import_path) + # sys.path.append(import_path) + # from resolvers import register_user_resolvers + + # register_user_resolvers(OmegaConf) + selected_targets = [] # loop through targets searchpath and load all targets @@ -130,7 +139,10 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo # resolve references / interpolate values try: - target_config = OmegaConf.to_object(OmegaConf.create(OmegaConf.to_object(target_config))) + OmegaConf.resolve(target_config) + + target_config = OmegaConf.to_object(target_config) + # target_config = OmegaConf.to_object(OmegaConf.create(OmegaConf.to_object(target_config))) except errors.OmegaConfBaseException as e: raise InventoryError(e.__context__) @@ -158,58 +170,40 @@ def fullkey(_node_: Node): return _node_._get_full_key("") -def name(name: str): - """resolver function, that returns a parameterized dependency""" - return {"name": name} - - def merge(*args): - """resolver function, that returns a parameterized dependency""" - merge = OmegaConf.merge(*args) - print(merge) + merge = OmegaConf.merge(*args, list_merge_mode=ListMergeMode.EXTEND) return merge -def add(o1, o2): - return o1 + o2 +def to_dict(input): + if not (isinstance(input, list) or OmegaConf.is_list(input)): + return input # not supported -def namespace(component: str): - return "${oc.select:parameters.components." + component + ".namespace}" + if not (isinstance(input[0], dict) or OmegaConf.is_dict(input[0])): + return input + return {key: item[key] for item in input for key in item} -def deployment(component: str): - cfg = { - "namespace": f"${{oc.select:parameters.{component}-config.namespace,{component}}}", - "image": f"${{oc.select:parameters.{component}-config.image,${{parameters.config.container.base_image}}}}", - } - cfg |= service(component) - return cfg +def to_list(input): + if isinstance(input, dict) or OmegaConf.is_dict(input): + return [{item[0]: item[1]} for item in input.items()] -def service(component: str): + return list(input) - source = { - "service": {"type": "", "selector": {"app": ""}}, - "ports": {"http": {"service_port": "187"}}, - } - def replace(input_dict, path=""): - processed_dict = {} +def namespace(component: str): + return "${oc.select:parameters.components." + component + ".namespace}" + - for key, value in input_dict.items(): - key_path = f"{path}.{key}" if path else key - if isinstance(value, dict): - processed_dict[key] = replace(value, path=key_path) - else: - search = f"parameters.{component}-config.{key_path}" - default = f"${{parameters.templates.deployment.{key_path}}}" if not value else value - processed_dict[key] = f"${{oc.select:{search},{default}}}" +def deployment(component: str): + return "${merge:${parameters.templates.deployment},${parameters." + component + "}}" - return processed_dict - return replace(source) +def copy(component: str, new_name): + return "${merge:${parameters.components." + component + "},${parameters." + new_name + "}}" def register_resolvers(): @@ -219,9 +213,43 @@ def register_resolvers(): OmegaConf.register_new_resolver("fullkey", fullkey) # kapitan helpers - OmegaConf.register_new_resolver("name", name) OmegaConf.register_new_resolver("merge", merge) - OmegaConf.register_new_resolver("add", add) + OmegaConf.register_new_resolver("dict", to_dict) + OmegaConf.register_new_resolver("list", to_list) OmegaConf.register_new_resolver("namespace", namespace) OmegaConf.register_new_resolver("deployment", deployment) - OmegaConf.register_new_resolver("service", service) + OmegaConf.register_new_resolver("copy", copy) + + +# def deployment(component: str): +# cfg = { +# "namespace": f"${{oc.select:parameters.{component}.namespace,{component}}}", +# "image": f"${{oc.select:parameters.{component}.image,${{parameters.config.container.base_image}}}}", +# } + +# cfg |= service(component) +# return cfg + + +# def service(component: str): + +# source = { +# "service": {"type": "", "selector": {"app": ""}}, +# "ports": {"http": {"service_port": "187"}}, +# } + +# def replace(input_dict, path=""): +# processed_dict = {} + +# for key, value in input_dict.items(): +# key_path = f"{path}.{key}" if path else key +# if isinstance(value, dict): +# processed_dict[key] = replace(value, path=key_path) +# else: +# search = f"parameters.{component}-config.{key_path}" +# default = f"${{parameters.templates.deployment.{key_path}}}" if not value else value +# processed_dict[key] = f"${{oc.select:{search},{default}}}" + +# return processed_dict + +# return replace(source) From 7dc758527f91b212cd8f9dd95ee3160e68ea6585 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:14:19 +0200 Subject: [PATCH 30/37] fix: namespace error with flag migrate --- kapitan/omegaconf_inv.py | 68 +++++++++++++++++++++++++++++++++++++--- kapitan/resources.py | 7 +++-- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index f413a1120..2af25929f 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -68,7 +68,7 @@ def inventory_omegaconf( name, config = load_target(target, classes_searchpath, ignore_class_notfound) inv["nodes"][name] = config except Exception as e: - logger.error(f"{target['name']}: {e}") + raise e # logger.error(f"{target['name']}: {e}") return inv @@ -132,8 +132,8 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo "name": { "full": target_name, "parts": target_name.split("."), - "path": target_name, - "short": target_name, + "path": target_name.replace(".", "/"), + "short": target_name.split(".")[-1], } } @@ -144,7 +144,7 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo target_config = OmegaConf.to_object(target_config) # target_config = OmegaConf.to_object(OmegaConf.create(OmegaConf.to_object(target_config))) except errors.OmegaConfBaseException as e: - raise InventoryError(e.__context__) + raise e # InventoryError(e.__context__) # obtain target name to insert in inv dict try: @@ -175,6 +175,11 @@ def merge(*args): return merge +def merge_replace(*args): + merge = OmegaConf.merge(*args, list_merge_mode=ListMergeMode.REPLACE) + return merge + + def to_dict(input): if not (isinstance(input, list) or OmegaConf.is_list(input)): @@ -194,6 +199,27 @@ def to_list(input): return list(input) +def relpath(path: str, _node_: Node): + + start = _node_._get_full_key("") + start = start.replace("[", ".") + + path_parts = path.split(".") + start_parts = start.split(".") + + while path_parts and start_parts and path_parts[0] == start_parts[0]: + path_parts.pop(0) + start_parts.pop(0) + + # Construct relative path + rel_parts = ["."] * (len(start_parts)) + reminder_path = ".".join(path_parts) + + rel_path = "".join(rel_parts) + reminder_path + + return f"${{{rel_path}}}" + + def namespace(component: str): return "${oc.select:parameters.components." + component + ".namespace}" @@ -206,6 +232,34 @@ def copy(component: str, new_name): return "${merge:${parameters.components." + component + "},${parameters." + new_name + "}}" +def helm_dep(name: str, source: str): + + return { + "type": "helm", + "output_path": f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}", + "source": source, + "version": f"${{parameters.{name}.chart_version}}", + "chart_name": f"${{parameters.{name}.chart_name}}", + } + + +def helm_input(name: str): + + return { + "input_type": "helm", + "input_paths": [ + f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}" + ], + "output_path": f"k8s/${{parameters.{name}.namespace}}", + "helm_params": { + "namespace": f"${{parameters.{name}.namespace}}", + "name": f"${{parameters.{name}.chart_name}}", + "output_file": f"{name}.yml", + }, + "helm_values": f"${{parameters.{name}.helm_values}}", + } + + def register_resolvers(): # utils OmegaConf.register_new_resolver("key", key) @@ -214,8 +268,14 @@ def register_resolvers(): # kapitan helpers OmegaConf.register_new_resolver("merge", merge) + OmegaConf.register_new_resolver("merge_replace", merge_replace) OmegaConf.register_new_resolver("dict", to_dict) OmegaConf.register_new_resolver("list", to_list) + OmegaConf.register_new_resolver("relpath", relpath) + OmegaConf.register_new_resolver("helm_dep", helm_dep) + OmegaConf.register_new_resolver("helm_input", helm_input) + + # kubernetes helpers OmegaConf.register_new_resolver("namespace", namespace) OmegaConf.register_new_resolver("deployment", deployment) OmegaConf.register_new_resolver("copy", copy) diff --git a/kapitan/resources.py b/kapitan/resources.py index c6bbc320f..e417e6f84 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -320,7 +320,10 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): # get parsed args from cached.py args = list(cached.args.values())[0] use_omegaconf = args.omegaconf - migrate_omegaconf = args.migrate + try: + migrate_omegaconf = args.migrate + except: + migrate_omegaconf = False if use_omegaconf: # show warning @@ -337,7 +340,7 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): try: inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) except errors.OmegaConfBaseException as e: - raise InventoryError(e) + raise e # InventoryError(e) else: logger.debug("Using reclass as inventory backend") inv = inventory_reclass(inventory_path, ignore_class_notfound) From 47b2c54c5009c0887a7db362238db2c50d156399 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:05:42 +0200 Subject: [PATCH 31/37] feat: add ability to define user resolvers in inventory --- kapitan/omegaconf_inv.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 2af25929f..e6693b166 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -27,13 +27,21 @@ def inventory_omegaconf( register_resolvers() - # user resolvers - # import_path = os.path.join(os.getcwd(), inventory_path) - # print(import_path) - # sys.path.append(import_path) - # from resolvers import register_user_resolvers + # import user resolvers + try: + import_path = os.path.join(os.getcwd(), inventory_path) + sys.path.append(import_path) + + from resolvers import pass_resolvers + + funcs = pass_resolvers() + + import resolvers - # register_user_resolvers(OmegaConf) + for name, func in funcs.items(): + OmegaConf.register_new_resolver(name, func) + except: + logger.warning("Couldnt import user resolvers") selected_targets = [] From b605005d00b906d55413073ba71dc5773113df4e Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:51:10 +0200 Subject: [PATCH 32/37] fix: user written resolvers replace system resolvers --- kapitan/omegaconf_inv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index e6693b166..779cea633 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -39,7 +39,7 @@ def inventory_omegaconf( import resolvers for name, func in funcs.items(): - OmegaConf.register_new_resolver(name, func) + OmegaConf.register_new_resolver(name, func, replace=True) except: logger.warning("Couldnt import user resolvers") From 3c616d58214e90eaab90ed3b1a724cc4287693fb Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:39:29 +0200 Subject: [PATCH 33/37] feat: restructure resolving and migrating --- kapitan/migrate_omegaconf.py | 117 ---------------- kapitan/omegaconf_inv.py | 260 ++++++++--------------------------- kapitan/resolvers.py | 158 +++++++++++++++++++++ kapitan/resources.py | 19 ++- 4 files changed, 225 insertions(+), 329 deletions(-) delete mode 100644 kapitan/migrate_omegaconf.py create mode 100644 kapitan/resolvers.py diff --git a/kapitan/migrate_omegaconf.py b/kapitan/migrate_omegaconf.py deleted file mode 100644 index 1c767039a..000000000 --- a/kapitan/migrate_omegaconf.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from ruamel.yaml import YAML -from pathlib import Path -from typing import Any -import regex as re - -REF_TOKEN = r"(? str: - inner_token = token[2:-1] - - if "parameters." in inner_token: - return token - - offset = 0 - matches = re.finditer(REF_TOKEN, inner_token) - for match in matches: - replaced = replace_token(match.group()) - inner_token = inner_token[: match.start() - offset] + replaced + inner_token[match.end() - offset :] - offset += len(match.group()) - len(replaced) - - inner_token = inner_token.replace(":", ".") - inner_token = "parameters." + inner_token - - token = "${" + inner_token + "}" - - return token - - -def replace_str(input: str) -> str: - offset = 0 - matches = re.finditer(REF_TOKEN, input) - for match in matches: - replaced = replace_token(match.group()) - input = input[: match.start() - offset] + replaced + input[match.end() - offset :] - offset += len(match.group()) - len(replaced) - - return input - - -# replace all references with OmegaConf syntax -def migrate_yaml_obj(yaml_obj: Any) -> Any: - # dictionary - if isinstance(yaml_obj, dict): - for k, v in yaml_obj.items(): - yaml_obj[k] = migrate_yaml_obj(v) - # list - elif isinstance(yaml_obj, list): - yaml_obj = [migrate_yaml_obj(item) for item in yaml_obj] - # string --> replace the references - elif isinstance(yaml_obj, str): - yaml_obj = replace_str(yaml_obj) - - return yaml_obj - - -def migrate_file(input_file: str) -> None: - # load the file - yaml = YAML(typ="rt") - yaml.preserve_quotes = True - - file_path = Path(input_file).resolve() - - try: - yaml_obj = yaml.load(file_path) - yaml_obj = migrate_yaml_obj(yaml_obj) - except: - print("ERROR in: ", file_path) - return - # yaml.dump(yaml_obj, file_path) - - yaml.indent(mapping=2, sequence=4, offset=2) - yaml.dump(yaml_obj, file_path) - - -def migrate(inv_path: str, output_path: str = "") -> None: - targets_path = os.path.join(inv_path, "targets") - classes_path = os.path.join(inv_path, "classes") - - for root, subdirs, files in os.walk(targets_path): - for target_file in files: - target_file = os.path.join(root, target_file) - _, ext = os.path.splitext(target_file) - - if ext not in (".yml", ".yaml"): - continue - - migrate_file(target_file) - - for root, subdirs, files in os.walk(classes_path): - for class_file in files: - class_file = os.path.join(root, class_file) - - _, ext = os.path.splitext(class_file) - - if ext not in (".yml", ".yaml"): - continue - - migrate_file(class_file) - - -# support running the file without kapitan -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: migrate_omegaconf.py INV_PATH") - sys.exit(1) - - inv_path = sys.argv[1] - - if not os.path.exists(inv_path): - print("Path does not exist") - - migrate(inv_path) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 779cea633..361fab114 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -3,10 +3,12 @@ # Copyright 2023 nexenio import logging import os -import sys + +import regex from kapitan.errors import InventoryError -from omegaconf import Node, OmegaConf, errors, ListMergeMode +from kapitan.resolvers import register_resolvers +from omegaconf import ListMergeMode, Node, OmegaConf, errors logger = logging.getLogger(__name__) @@ -25,23 +27,7 @@ def inventory_omegaconf( targets_searchpath = os.path.join(inventory_path, "targets") classes_searchpath = os.path.join(inventory_path, "classes") - register_resolvers() - - # import user resolvers - try: - import_path = os.path.join(os.getcwd(), inventory_path) - sys.path.append(import_path) - - from resolvers import pass_resolvers - - funcs = pass_resolvers() - - import resolvers - - for name, func in funcs.items(): - OmegaConf.register_new_resolver(name, func, replace=True) - except: - logger.warning("Couldnt import user resolvers") + register_resolvers(inventory_path) selected_targets = [] @@ -68,6 +54,7 @@ def inventory_omegaconf( selected_targets.append({"name": target_name, "path": target_path}) + # using nodes for reclass legacy code inv = {"nodes": {}} # load targets @@ -76,7 +63,7 @@ def inventory_omegaconf( name, config = load_target(target, classes_searchpath, ignore_class_notfound) inv["nodes"][name] = config except Exception as e: - raise e # logger.error(f"{target['name']}: {e}") + raise InventoryError(f"{target['name']}: {e}") return inv @@ -90,25 +77,27 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo target_path = target["path"] target_config = OmegaConf.load(target_path) - - target_config_classes = target_config.pop("classes", []) + target_config_classes = target_config.get("classes", []) + target_config_parameters = OmegaConf.create(target_config.get("parameters", {})) + target_config = {} # load classes for targets for class_name in target_config_classes: - # resolve class paths + # resolve class path class_path = os.path.join(classes_searchpath, *class_name.split(".")) - # search for init file - if os.path.isdir(class_path): + if os.path.isfile(class_path + ".yml"): + class_path += ".yml" + elif os.path.isdir(class_path): + # search for init file init_path = os.path.join(classes_searchpath, *class_name.split("."), "init") + ".yml" if os.path.isfile(init_path): class_path = init_path + elif ignore_class_notfound: + logger.debug(f"Could not find {class_path}") + continue else: - class_path += ".yml" - - if not os.path.isfile(class_path): - if not ignore_class_notfound: - raise InventoryError(f"Class {class_name} not found.") + raise InventoryError(f"Class {class_name} not found.") # load classes recursively class_config = OmegaConf.load(class_path) @@ -121,22 +110,21 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo target_config_classes.append(new) + class_config_parameters = OmegaConf.create(class_config.get("parameters", {})) + # merge target with loaded classes - if target_config.get("parameters"): - target_config = OmegaConf.unsafe_merge( - class_config, target_config, list_merge_mode=ListMergeMode.EXTEND + if target_config_parameters: + target_config_parameters = OmegaConf.unsafe_merge( + class_config_parameters, target_config_parameters, list_merge_mode=ListMergeMode.EXTEND ) else: - target_config = class_config + target_config_parameters = class_config_parameters - if not target_config: + if not target_config_parameters: raise InventoryError("empty target") - if not target_config.get("parameters"): - raise InventoryError("target has no parameters") - - # append meta data _reclass_ (legacy) - target_config["parameters"]["_reclass_"] = { + # append meta data (legacy: _reclass_) + target_config_parameters["_reclass_"] = { "name": { "full": target_name, "parts": target_name.split("."), @@ -146,13 +134,8 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo } # resolve references / interpolate values - try: - OmegaConf.resolve(target_config) - - target_config = OmegaConf.to_object(target_config) - # target_config = OmegaConf.to_object(OmegaConf.create(OmegaConf.to_object(target_config))) - except errors.OmegaConfBaseException as e: - raise e # InventoryError(e.__context__) + OmegaConf.resolve(target_config_parameters) + target_config["parameters"] = OmegaConf.to_object(target_config_parameters) # obtain target name to insert in inv dict try: @@ -163,161 +146,34 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo return target_name, target_config -def key(_node_: Node): - """resolver function, that returns the name of its parent key""" - return _node_._key() - - -def parentkey(_parent_: Node): - """resolver function, that returns the name of its parent key""" - return _parent_._key() - - -def fullkey(_node_: Node): - """resolver function, that returns the full name of its parent key""" - return _node_._get_full_key("") - - -def merge(*args): - merge = OmegaConf.merge(*args, list_merge_mode=ListMergeMode.EXTEND) - return merge - - -def merge_replace(*args): - merge = OmegaConf.merge(*args, list_merge_mode=ListMergeMode.REPLACE) - return merge - - -def to_dict(input): - - if not (isinstance(input, list) or OmegaConf.is_list(input)): - return input # not supported - - if not (isinstance(input[0], dict) or OmegaConf.is_dict(input[0])): - return input - - return {key: item[key] for item in input for key in item} - - -def to_list(input): - - if isinstance(input, dict) or OmegaConf.is_dict(input): - return [{item[0]: item[1]} for item in input.items()] - - return list(input) - +def migrate(inventory_path: str) -> None: + """migrates all .yml/.yaml files in the given path to omegaconfs syntax""" -def relpath(path: str, _node_: Node): + for root, subdirs, files in os.walk(inventory_path): + for file in files: + file = os.path.join(root, file) + name, ext = os.path.splitext(file) - start = _node_._get_full_key("") - start = start.replace("[", ".") - - path_parts = path.split(".") - start_parts = start.split(".") - - while path_parts and start_parts and path_parts[0] == start_parts[0]: - path_parts.pop(0) - start_parts.pop(0) - - # Construct relative path - rel_parts = ["."] * (len(start_parts)) - reminder_path = ".".join(path_parts) - - rel_path = "".join(rel_parts) + reminder_path - - return f"${{{rel_path}}}" - - -def namespace(component: str): - return "${oc.select:parameters.components." + component + ".namespace}" - - -def deployment(component: str): - return "${merge:${parameters.templates.deployment},${parameters." + component + "}}" - - -def copy(component: str, new_name): - return "${merge:${parameters.components." + component + "},${parameters." + new_name + "}}" - - -def helm_dep(name: str, source: str): - - return { - "type": "helm", - "output_path": f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}", - "source": source, - "version": f"${{parameters.{name}.chart_version}}", - "chart_name": f"${{parameters.{name}.chart_name}}", - } - - -def helm_input(name: str): - - return { - "input_type": "helm", - "input_paths": [ - f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}" - ], - "output_path": f"k8s/${{parameters.{name}.namespace}}", - "helm_params": { - "namespace": f"${{parameters.{name}.namespace}}", - "name": f"${{parameters.{name}.chart_name}}", - "output_file": f"{name}.yml", - }, - "helm_values": f"${{parameters.{name}.helm_values}}", - } - - -def register_resolvers(): - # utils - OmegaConf.register_new_resolver("key", key) - OmegaConf.register_new_resolver("parentkey", parentkey) - OmegaConf.register_new_resolver("fullkey", fullkey) - - # kapitan helpers - OmegaConf.register_new_resolver("merge", merge) - OmegaConf.register_new_resolver("merge_replace", merge_replace) - OmegaConf.register_new_resolver("dict", to_dict) - OmegaConf.register_new_resolver("list", to_list) - OmegaConf.register_new_resolver("relpath", relpath) - OmegaConf.register_new_resolver("helm_dep", helm_dep) - OmegaConf.register_new_resolver("helm_input", helm_input) - - # kubernetes helpers - OmegaConf.register_new_resolver("namespace", namespace) - OmegaConf.register_new_resolver("deployment", deployment) - OmegaConf.register_new_resolver("copy", copy) - - -# def deployment(component: str): -# cfg = { -# "namespace": f"${{oc.select:parameters.{component}.namespace,{component}}}", -# "image": f"${{oc.select:parameters.{component}.image,${{parameters.config.container.base_image}}}}", -# } - -# cfg |= service(component) -# return cfg - - -# def service(component: str): - -# source = { -# "service": {"type": "", "selector": {"app": ""}}, -# "ports": {"http": {"service_port": "187"}}, -# } - -# def replace(input_dict, path=""): -# processed_dict = {} - -# for key, value in input_dict.items(): -# key_path = f"{path}.{key}" if path else key -# if isinstance(value, dict): -# processed_dict[key] = replace(value, path=key_path) -# else: -# search = f"parameters.{component}-config.{key_path}" -# default = f"${{parameters.templates.deployment.{key_path}}}" if not value else value -# processed_dict[key] = f"${{oc.select:{search},{default}}}" - -# return processed_dict + if ext not in (".yml", ".yaml"): + continue -# return replace(source) + try: + with open(file, "r+") as file: + content = file.read() + file.seek(0) + + # replace_colons_in_tags + updated_content = regex.sub( + r"(? None: + """register pre-defined and user-defined resolvers""" + + # yaml key utility functions + OmegaConf.register_new_resolver("key", key) + OmegaConf.register_new_resolver("parentkey", parentkey) + OmegaConf.register_new_resolver("fullkey", fullkey) + OmegaConf.register_new_resolver("relpath", relpath) + + # yaml object utility functions + OmegaConf.register_new_resolver("tag", escape_tag) + OmegaConf.register_new_resolver("merge", merge) + OmegaConf.register_new_resolver("dict", to_dict) + OmegaConf.register_new_resolver("list", to_list) + + # kapitan helpers / templates + OmegaConf.register_new_resolver("helm_dep", helm_dep) + OmegaConf.register_new_resolver("helm_input", helm_input) + + # user defined resolvers + user_resolver_file = os.path.join(inventory_path, "resolvers.py") + if os.path.exists(user_resolver_file): + try: + register_user_resolvers(inventory_path) + except: + logger.debug(f"Couldn't import {os.join(inventory_path, 'resolvers.py')}") + + +def register_user_resolvers(inventory_path: str) -> None: + """import user resolvers specified in inventory/resolvers.py""" + try: + import_path = os.path.join(os.getcwd(), inventory_path) + sys.path.append(import_path) + from resolvers import pass_resolvers + + funcs = pass_resolvers() + except: + logger.warning("resolvers.py must contain function 'pass_resolvers()'") + return + + if not isinstance(funcs, dict): + logger.warning("pass_resolvers() should return a dict") + return + + import resolvers + + for name, func in funcs.items(): + try: + OmegaConf.register_new_resolver(name, func, replace=True) + except: + logger.warning(f"Could not load resolver {name}") diff --git a/kapitan/resources.py b/kapitan/resources.py index e417e6f84..ba193f606 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -14,6 +14,7 @@ import logging import os import sys +import time from functools import partial import jsonschema @@ -25,10 +26,8 @@ import kapitan.cached as cached from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError -from kapitan.migrate_omegaconf import migrate -from kapitan.omegaconf_inv import inventory_omegaconf +from kapitan.omegaconf_inv import inventory_omegaconf, migrate from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -from omegaconf import errors logger = logging.getLogger(__name__) @@ -332,15 +331,15 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): # migrate a reclass inventory to omegaConf if migrate_omegaconf: - output_path = inventory_path - if not os.path.exists(output_path): - os.mkdir(output_path) - migrate(inventory_path, output_path) - logger.info(f"Migrated inventory to OmegaConf in {output_path}") + migrate_start = time.time() + migrate(inventory_path) + logger.info("Migrated inventory to OmegaConf (%.2fs)", time.time() - migrate_start) try: inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) - except errors.OmegaConfBaseException as e: - raise e # InventoryError(e) + except Exception as e: + if not migrate_omegaconf: + logger.warning("Make sure to migrate your inventory using --migrate") + raise InventoryError(e) else: logger.debug("Using reclass as inventory backend") inv = inventory_reclass(inventory_path, ignore_class_notfound) From 08a21947d48d91cf99ec8fa97f2bc6ccd81f888d Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:47:14 +0200 Subject: [PATCH 34/37] chore: remove ruamel-yaml and add omegaconf in poetry-file --- poetry.lock | 99 ++++++++++++++++---------------------------------- pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 68 deletions(-) diff --git a/poetry.lock b/poetry.lock index 96bf2936e..3b321a7bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,6 +12,17 @@ files = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] + [[package]] name = "attrs" version = "22.2.0" @@ -817,6 +828,26 @@ portalocker = [ {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, ] +[[package]] +name = "omegaconf" +version = "2.4.0.dev0" +description = "A flexible configuration library" +category = "main" +optional = false +python-versions = ">=3.6" +files = [] +develop = false + +[package.dependencies] +antlr4-python3-runtime = ">=4.9.0,<4.10.0" +PyYAML = ">=5.1.0" + +[package.source] +type = "git" +url = "https://github.com/omry/omegaconf.git" +reference = "HEAD" +resolved_reference = "7dae67e4a3869b584fd6d6408b2f916c176755db" + [[package]] name = "packaging" version = "23.0" @@ -1339,72 +1370,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" -[[package]] -name = "ruamel-yaml" -version = "0.17.31" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" -optional = false -python-versions = ">=3" -files = [ - {file = "ruamel.yaml-0.17.31-py3-none-any.whl", hash = "sha256:3cf153f0047ced526e723097ac615d3009371779432e304dbd5596b6f3a4c777"}, - {file = "ruamel.yaml-0.17.31.tar.gz", hash = "sha256:098ed1eb6d338a684891a72380277c1e6fc4d4ae0e120de9a447275056dda335"}, -] - -[package.dependencies] -"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.12\""} - -[package.extras] -docs = ["ryd"] -jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] - -[[package]] -name = "ruamel-yaml-clib" -version = "0.2.7" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" -optional = false -python-versions = ">=3.5" -files = [ - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5859983f26d8cd7bb5c287ef452e8aacc86501487634573d260968f753e1d71"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:debc87a9516b237d0466a711b18b6ebeb17ba9f391eb7f91c649c5c4ec5006c7"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:df5828871e6648db72d1c19b4bd24819b80a755c4541d3409f0f7acd0f335c80"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:efa08d63ef03d079dcae1dfe334f6c8847ba8b645d08df286358b1f5293d24ab"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, - {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4b3a93bb9bc662fc1f99c5c3ea8e623d8b23ad22f861eb6fce9377ac07ad6072"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-macosx_12_0_arm64.whl", hash = "sha256:a234a20ae07e8469da311e182e70ef6b199d0fbeb6c6cc2901204dd87fb867e8"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:15910ef4f3e537eea7fe45f8a5d19997479940d9196f357152a09031c5be59f3"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:370445fd795706fd291ab00c9df38a0caed0f17a6fb46b0f607668ecb16ce763"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win32.whl", hash = "sha256:ecdf1a604009bd35c674b9225a8fa609e0282d9b896c03dd441a91e5f53b534e"}, - {file = "ruamel.yaml.clib-0.2.7-cp36-cp36m-win_amd64.whl", hash = "sha256:f34019dced51047d6f70cb9383b2ae2853b7fc4dce65129a5acd49f4f9256646"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2aa261c29a5545adfef9296b7e33941f46aa5bbd21164228e833412af4c9c75f"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f01da5790e95815eb5a8a138508c01c758e5f5bc0ce4286c4f7028b8dd7ac3d0"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:40d030e2329ce5286d6b231b8726959ebbe0404c92f0a578c0e2482182e38282"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c3ca1fbba4ae962521e5eb66d72998b51f0f4d0f608d3c0347a48e1af262efa7"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:7bdb4c06b063f6fd55e472e201317a3bb6cdeeee5d5a38512ea5c01e1acbdd93"}, - {file = "ruamel.yaml.clib-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:be2a7ad8fd8f7442b24323d24ba0b56c51219513cfa45b9ada3b87b76c374d4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91a789b4aa0097b78c93e3dc4b40040ba55bef518f84a40d4442f713b4094acb"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:99e77daab5d13a48a4054803d052ff40780278240a902b880dd37a51ba01a307"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3243f48ecd450eddadc2d11b5feb08aca941b5cd98c9b1db14b2fd128be8c697"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8831a2cedcd0f0927f788c5bdf6567d9dc9cc235646a434986a852af1cb54b4b"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win32.whl", hash = "sha256:3110a99e0f94a4a3470ff67fc20d3f96c25b13d24c6980ff841e82bafe827cac"}, - {file = "ruamel.yaml.clib-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:92460ce908546ab69770b2e576e4f99fbb4ce6ab4b245345a3869a0a0410488f"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc0667c1eb8f83a3752b71b9c4ba55ef7c7058ae57022dd9b29065186a113d9"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:4a4d8d417868d68b979076a9be6a38c676eca060785abaa6709c7b31593c35d1"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf9a6bc4a0221538b1a7de3ed7bca4c93c02346853f44e1cd764be0023cd3640"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a7b301ff08055d73223058b5c46c55638917f04d21577c95e00e0c4d79201a6b"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win32.whl", hash = "sha256:d5e51e2901ec2366b79f16c2299a03e74ba4531ddcfacc1416639c557aef0ad8"}, - {file = "ruamel.yaml.clib-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:184faeaec61dbaa3cace407cffc5819f7b977e75360e8d5ca19461cd851a5fc5"}, - {file = "ruamel.yaml.clib-0.2.7.tar.gz", hash = "sha256:1f08fd5a2bea9c4180db71678e850b995d2a5f4537be0e94557668cf0f5f9497"}, -] - [[package]] name = "s3transfer" version = "0.6.0" @@ -1594,4 +1559,4 @@ test = ["docker"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "a4295ac23e667931d5858408ee67bcffa9cb58d1520b794a4e544a9b6a15f6eb" +content-hash = "e8c1a9fdd874ef3e43ed6e11e70445e23c2a03c2b5a2245bf9419077ee81daaa" diff --git a/pyproject.toml b/pyproject.toml index 34dc83f3d..6ec7ff1fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ jsonschema = "^4.17.3" kadet = "^0.2.2" python-gnupg = "^0.4.7" pyyaml = "^6.0" +omegaconf = { git = "https://github.com/omry/omegaconf.git" } requests = "^2.28.2" six = "^1.16.0" toml = "^0.10.2" @@ -63,7 +64,6 @@ packaging = "^23.0" typing-extensions = "^4.0.0" gojsonnet = { version = "^0.17.0", optional = true } docker = { version = "^5.0.0", optional = true } -ruamel-yaml = "^0.17.31" regex = "^2023.5.5" [tool.poetry.extras] From ba04981aecb74d452ed75af962fcda831d9ffe68 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:23:19 +0200 Subject: [PATCH 35/37] fix: resolver escape_tag was missing braces --- kapitan/resolvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/resolvers.py b/kapitan/resolvers.py index feddb85bc..85ba3a4c9 100644 --- a/kapitan/resolvers.py +++ b/kapitan/resolvers.py @@ -28,7 +28,7 @@ def fullkey(_node_: Node): def escape_tag(input: str): """resolver function, that returns an escaped tag with the input""" - return f"\\${{input}}" + return f"\\${{{input}}}" def merge(*args): From 78e83fff41d28831c2cbac355358f82c6305d17f Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Thu, 13 Jul 2023 14:54:23 +0200 Subject: [PATCH 36/37] fix: correct wrong behavior of resolver `tag` --- kapitan/omegaconf_inv.py | 25 ++++++++++++++++++++++--- kapitan/resolvers.py | 13 +++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index 361fab114..0a2eb7dbc 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -3,6 +3,7 @@ # Copyright 2023 nexenio import logging import os +import time import regex @@ -57,11 +58,16 @@ def inventory_omegaconf( # using nodes for reclass legacy code inv = {"nodes": {}} + # prepare logging + logger.info(f"Found {len(selected_targets)} targets") + # load targets for target in selected_targets: try: + # start = time.time() name, config = load_target(target, classes_searchpath, ignore_class_notfound) inv["nodes"][name] = config + # print(time.time() - start) except Exception as e: raise InventoryError(f"{target['name']}: {e}") @@ -81,11 +87,18 @@ def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bo target_config_parameters = OmegaConf.create(target_config.get("parameters", {})) target_config = {} + classes_redundancy_check = set() + # load classes for targets for class_name in target_config_classes: # resolve class path class_path = os.path.join(classes_searchpath, *class_name.split(".")) + if class_path in classes_redundancy_check: + continue + + classes_redundancy_check.add(class_path) + if os.path.isfile(class_path + ".yml"): class_path += ".yml" elif os.path.isdir(class_path): @@ -162,16 +175,22 @@ def migrate(inventory_path: str) -> None: content = file.read() file.seek(0) - # replace_colons_in_tags + # replace colons in tags and replace _reclass_ with _meta_ updated_content = regex.sub( r"(? Date: Thu, 13 Jul 2023 14:54:49 +0200 Subject: [PATCH 37/37] feat: prepare support for lint --- kapitan/resources.py | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/kapitan/resources.py b/kapitan/resources.py index ba193f606..b573ba715 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -335,14 +335,18 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): migrate(inventory_path) logger.info("Migrated inventory to OmegaConf (%.2fs)", time.time() - migrate_start) try: + # inv_start = time.time() inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) + # logger.info("REAL_TIME (%.2fs)", time.time() - inv_start) except Exception as e: if not migrate_omegaconf: logger.warning("Make sure to migrate your inventory using --migrate") raise InventoryError(e) else: logger.debug("Using reclass as inventory backend") + # inv_start = time.time() inv = inventory_reclass(inventory_path, ignore_class_notfound) + # logger.info("REAL_TIME (%.2fs)", time.time() - inv_start) cached.inv = inv return inv diff --git a/pyproject.toml b/pyproject.toml index 6ec7ff1fa..98b180c3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ jsonschema = "^4.17.3" kadet = "^0.2.2" python-gnupg = "^0.4.7" pyyaml = "^6.0" -omegaconf = { git = "https://github.com/omry/omegaconf.git" } +omegaconf = { git = "https://github.com/neXenio/omegaconf.git", branch = "dev" } requests = "^2.28.2" six = "^1.16.0" toml = "^0.10.2"