Cleanup config file loader (#208)

* fix config file loader

* prune nonetype values from config dict

fixes default values not initialising properly

* Utils: Shrink None removal function

It is more concise to use a list and dict collection if necessary
rather than iterating through and checking each value. Tested and
works with Tabby's cases.

Signed-off-by: kingbri <[email protected]>

---------

Signed-off-by: kingbri <[email protected]>
Co-authored-by: kingbri <[email protected]>

common/sampling.py

@@ -9,7 +9,7 @@
 from pydantic import AliasChoices, BaseModel, Field
 from typing import Dict, List, Optional, Union
 
-from common.utils import unwrap, prune_dict
+from common.utils import filter_none_values, unwrap
 
 
 # Common class for sampler params
@@ -403,7 +403,7 @@ def overrides_from_dict(new_overrides: dict):
     """Wrapper function to update sampler overrides"""
 
     if isinstance(new_overrides, dict):
-        overrides_container.overrides = prune_dict(new_overrides)
+        overrides_container.overrides = filter_none_values(new_overrides)
     else:
         raise TypeError("New sampler overrides must be a dict!")
 

common/tabby_config.py

@@ -11,7 +11,7 @@
 from ruamel.yaml.scalarstring import PreservedScalarString
 
 from common.config_models import BaseConfigModel, TabbyConfigModel
-from common.utils import merge_dicts, unwrap
+from common.utils import merge_dicts, filter_none_values, unwrap
 
 yaml = YAML(typ=["rt", "safe"])
 
@@ -33,6 +33,9 @@ def load(self, arguments: Optional[dict] = None):
         if not arguments_dict.get("actions"):
             configs.insert(0, self._from_file(pathlib.Path("config.yml")))
 
+        # Remove None (aka unset) values from the configs and merge them together
+        # This should be less expensive than pruning the entire merged dictionary
+        configs = filter_none_values(configs)
         merged_config = merge_dicts(*configs)
 
         # validate and update config
@@ -135,7 +138,7 @@ def _from_args(self, args: dict):
         """loads config from the provided arguments"""
         config = {}
 
-        config_override = args.get("options", {}).get("config", None)
+        config_override = args.get("config", {}).get("config", None)
         if config_override:
             logger.info("Config file override detected in args.")
             config = self._from_file(pathlib.Path(config_override))

common/utils.py

@@ -19,10 +19,17 @@ def coalesce(*args):
     return next((arg for arg in args if arg is not None), None)
 
 
-def prune_dict(input_dict: Dict) -> Dict:
-    """Trim out instances of None from a dictionary."""
-
-    return {k: v for k, v in input_dict.items() if v is not None}
+def filter_none_values(collection: Union[dict, list]) -> Union[dict, list]:
+    """Remove None values from a collection."""
+
+    if isinstance(collection, dict):
+        return {
+            k: filter_none_values(v) for k, v in collection.items() if v is not None
+        }
+    elif isinstance(collection, list):
+        return [filter_none_values(i) for i in collection if i is not None]
+    else:
+        return collection
 
 
 def merge_dict(dict1: Dict, dict2: Dict) -> Dict: