Merge pull request #762 from valory-xyz/release/1.58.0

Release `v1.58.0`

.spelling

@@ -3,6 +3,7 @@
 # global dictionary is at the start, file overrides afterwards
 # one word per line, to define a file override use ' - filename'
 # where filename is relative to this configuration file
+fraxtal
 Flashbots
 liveness
 pre-shared

HISTORY.md

@@ -1,5 +1,10 @@
 # Release History - open AEA
 
+## 1.58.0 (2024-10-03)
+
+AEA:
+- Adds support for dictionary overrides #750, #761
+
 ## 1.57.0 (2024-09-24)
 
 AEA:

SECURITY.md

@@ -8,8 +8,8 @@ The following table shows which versions of `open-aea` are currently being suppo
 
 | Version    | Supported          |
 |------------| ------------------ |
-| `1.57.x`   | :white_check_mark: |
-| `< 1.57.0` | :x:                |
+| `1.58.x`   | :white_check_mark: |
+| `< 1.58.0` | :x:                |
 
 ## Reporting a Vulnerability
 

aea/__version__.py

@@ -23,7 +23,7 @@
 __title__ = "open-aea"
 __description__ = "Open Autonomous Economic Agent framework (without vendor lock-in)"
 __url__ = "https://github.com/valory-xyz/open-aea.git"
-__version__ = "1.57.0"
+__version__ = "1.58.0"
 __author__ = "Valory AG"
 __license__ = "Apache-2.0"
 __copyright__ = "2021 Valory AG, 2019 Fetch.AI Limited"

aea/configurations/validation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,11 +18,13 @@
 #
 # ------------------------------------------------------------------------------
 """Implementation of the configuration validation."""
+
 import inspect
 import json
 import os
 from collections import OrderedDict
 from copy import deepcopy
+from functools import reduce
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
@@ -36,7 +38,7 @@
 from aea.configurations.constants import AGENT
 from aea.configurations.data_types import ComponentId, ComponentType, PublicId
 from aea.exceptions import AEAValidationError
-from aea.helpers.base import dict_to_path_value
+from aea.helpers.base import dict_to_path_value, update_nested_dict
 from aea.helpers.env_vars import is_env_variable
 from aea.helpers.io import open_file
 
@@ -295,12 +297,26 @@ def validate_data_with_pattern(
         excludes_: List[Tuple[str]] = []
     else:
         excludes_ = excludes
-    pattern_path_value = {
+    original_config = {
         tuple(path): value for path, value in dict_to_path_value(pattern)
     }
-    data_path_value = {tuple(path): value for path, value in dict_to_path_value(data)}
+    overrides = {tuple(path): value for path, value in dict_to_path_value(data)}
     errors = []
 
+    # this is a workaround to fix the type of numeric keys as they can only be represented as strs in the json overrides
+    for path in original_config:
+        path_as_str = tuple(map(str, path))
+        if path_as_str in overrides and path not in overrides:
+            value = overrides[path_as_str]
+            del overrides[path_as_str]
+            up_to_last_key = data
+            for key in path_as_str[:-1]:
+                up_to_last_key = up_to_last_key[key]
+            del up_to_last_key[path_as_str[-1]]
+            overrides[path] = value
+            vals = reduce(lambda d, key: {key: d}, reversed(path), value)
+            update_nested_dict(data, vals)
+
     def check_excludes(path: Tuple[str, ...]) -> bool:
         for exclude in excludes_:
             if len(exclude) > len(path):  # pragma: nocover
@@ -315,25 +331,25 @@ def is_a_dict_override(path: Tuple[str, ...]) -> bool:
         flag = False
         while len(path) > 0:
             path = path[:-1]
-            if path in pattern_path_value:
-                pattern_value = pattern_path_value[path]
+            if path in original_config:
+                pattern_value = original_config[path]
                 flag = isinstance(pattern_value, OrderedDict)
                 break
         return flag
 
-    for path, new_value in data_path_value.items():
+    for path, new_value in overrides.items():
         if check_excludes(path):
             continue
 
-        if path not in pattern_path_value:
+        if path not in original_config:
             if not is_a_dict_override(path=(*path,)):
                 errors.append(
                     f"Attribute `{'.'.join(path)}` is not allowed to be updated!"
                 )
 
             continue
 
-        pattern_value = pattern_path_value[path]
+        pattern_value = original_config[path]
 
         if pattern_value is None:
             # not possible to determine data type for optional value not set

aea/helpers/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -711,10 +711,14 @@ def dict_to_path_value(
     """Convert dict to sequence of terminal path build of  keys and value."""
     path = path or []
     for key, value in data.items():
+        # terminal value
         if isinstance(value, Mapping) and value:
-            # terminal value
+            # yielding here allows for higher level dict overriding
+            yield path + [key], value
+            # recursing to the next level of the dict
             for p, v in dict_to_path_value(value, path + [key]):
                 yield p, v
+        # non-terminal value
         else:
             yield path + [key], value
 
@@ -1087,3 +1091,13 @@ def prepend_if_not_absolute(path: PathLike, prefix: PathLike) -> PathLike:
     :return: the same path if absolute, else the prepended path.
     """
     return path if Path(path).is_absolute() else Path(prefix) / path
+
+
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict:
+    """Update a nested dictionary."""
+    for key, value in nested_update.items():
+        if isinstance(value, dict):
+            dict_[key] = update_nested_dict(dict_.get(key, {}), value)
+        else:
+            dict_[key] = value
+    return dict_

aea/helpers/env_vars.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2019 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -34,17 +34,47 @@
 
 
 ENV_VARIABLE_RE = re.compile(r"^\$\{(([A-Z0-9_]+):?)?([a-z]+)?(:(.+))?}$")
+MODELS = "models"
+ARGS = "args"
+ARGS_LEVEL_FROM_MODELS = 2
+ARG_LEVEL_FROM_MODELS = ARGS_LEVEL_FROM_MODELS + 1
+RESTRICTION_EXCEPTIONS = frozenset({"setup", "genesis_config"})
 
 
 def is_env_variable(value: Any) -> bool:
     """Check is variable string with env variable pattern."""
     return isinstance(value, str) and bool(ENV_VARIABLE_RE.match(value))
 
 
-def export_path_to_env_var_string(export_path: List[str]) -> str:
-    """Conver export path to environment variable string."""
+def restrict_model_args(export_path: List[str]) -> Tuple[List[str], List[str]]:
+    """Do not allow more levels than one for a model's argument."""
+    restricted = []
+    result = []
+    for i, current_path in enumerate(export_path):
+        result.append(current_path)
+        args_level = i + ARGS_LEVEL_FROM_MODELS
+        arg_level = i + ARG_LEVEL_FROM_MODELS
+        if (
+            current_path == MODELS
+            and arg_level < len(export_path)
+            and export_path[args_level] == ARGS
+            and export_path[arg_level] not in RESTRICTION_EXCEPTIONS
+        ):
+            # do not allow more levels than one for a model's argument
+            arg_content_level = arg_level + 1
+            result.extend(export_path[i + 1 : arg_content_level])
+            # store the restricted part of the path
+            for j in range(arg_content_level, len(export_path)):
+                restricted.append(export_path[j])
+            break
+    return restricted, result
+
+
+def export_path_to_env_var_string(export_path: List[str]) -> Tuple[List[str], str]:
+    """Convert export path to environment variable string."""
+    restricted, export_path = restrict_model_args(export_path)
     env_var_string = "_".join(map(str, export_path))
-    return env_var_string.upper()
+    return restricted, env_var_string.upper()
 
 
 NotSet = object()
@@ -149,7 +179,7 @@ def apply_env_variables(
             data,
             env_variables,
             default_value,
-            default_var_name=export_path_to_env_var_string(export_path=path),
+            default_var_name=export_path_to_env_var_string(export_path=path)[1],
         )
 
     return data
@@ -242,35 +272,76 @@ def is_strict_list(data: Union[List, Tuple]) -> bool:
     return is_strict
 
 
+def list_to_nested_dict(lst: list, val: Any) -> dict:
+    """Convert a list to a nested dict."""
+    nested_dict = val
+    for item in reversed(lst):
+        nested_dict = {item: nested_dict}
+    return nested_dict
+
+
+def ensure_dict(dict_: Dict[str, Union[dict, str]]) -> dict:
+    """Return the given dictionary converting any values which are json strings as dicts."""
+    return {k: json.loads(v) for k, v in dict_.items() if isinstance(v, str)}
+
+
+def ensure_json_content(dict_: dict) -> dict:
+    """Return the given dictionary converting any nested dictionary values as json strings."""
+    return {k: json.dumps(v) for k, v in dict_.items() if isinstance(v, dict)}
+
+
+def merge_dicts(a: dict, b: dict) -> dict:
+    """Merge two dictionaries."""
+    # shallow copy of `a`
+    merged = {**a}
+    for key, value in b.items():
+        if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
+            # recursively merge nested dictionaries
+            merged[key] = merge_dicts(merged[key], value)
+        else:
+            # if not a nested dictionary, just take the value from `b`
+            merged[key] = value
+    return merged
+
+
 def generate_env_vars_recursively(
     data: Union[Dict, List],
     export_path: List[str],
 ) -> Dict:
     """Generate environment variables recursively."""
-    env_var_dict = {}
+    env_var_dict: Dict[str, Any] = {}
 
     if isinstance(data, dict):
         for key, value in data.items():
-            env_var_dict.update(
-                generate_env_vars_recursively(
-                    data=value,
-                    export_path=[*export_path, key],
-                )
+            res = generate_env_vars_recursively(
+                data=value,
+                export_path=[*export_path, key],
             )
+            if res:
+                env_var = list(res.keys())[0]
+                if env_var in env_var_dict:
+                    dicts = (ensure_dict(dict_) for dict_ in (env_var_dict, res))
+                    res = ensure_json_content(merge_dicts(*dicts))
+            env_var_dict.update(res)
     elif isinstance(data, list):
         if is_strict_list(data=data):
-            env_var_dict[
-                export_path_to_env_var_string(export_path=export_path)
-            ] = json.dumps(data, separators=(",", ":"))
+            restricted, path = export_path_to_env_var_string(export_path=export_path)
+            if restricted:
+                env_var_dict[path] = json.dumps(list_to_nested_dict(restricted, data))
+            else:
+                env_var_dict[path] = json.dumps(data, separators=(",", ":"))
         else:
             for key, value in enumerate(data):
-                env_var_dict.update(
-                    generate_env_vars_recursively(
-                        data=value,
-                        export_path=[*export_path, key],
-                    )
+                res = generate_env_vars_recursively(
+                    data=value,
+                    export_path=[*export_path, key],
                 )
+                env_var_dict.update(res)
     else:
-        env_var_dict[export_path_to_env_var_string(export_path=export_path)] = data
+        restricted, path = export_path_to_env_var_string(export_path=export_path)
+        if restricted:
+            env_var_dict[path] = json.dumps(list_to_nested_dict(restricted, data))
+        else:
+            env_var_dict[path] = data
 
     return env_var_dict

deploy-image/Dockerfile

@@ -16,7 +16,7 @@ RUN apk add --no-cache go
 
 # aea installation
 RUN pip install --upgrade pip
-RUN pip install --upgrade --force-reinstall open-aea[all]==1.57.0 "open-aea-cli-ipfs<2.0.0,>=1.57.0"
+RUN pip install --upgrade --force-reinstall open-aea[all]==1.58.0 "open-aea-cli-ipfs<2.0.0,>=1.58.0"
 
 # directories and aea cli config
 WORKDIR /home/agents

deploy-image/README.md

@@ -11,7 +11,7 @@ The example uses the `fetchai/my_first_aea` project. You will likely want to mod
 Install subversion, then download the example directory to your local working directory
 
 ``` bash
-svn checkout https://github.com/valory-xyz/open-aea/tags/v1.57.0/packages packages
+svn checkout https://github.com/valory-xyz/open-aea/tags/v1.58.0/packages packages
 ```
 
 ### Modify scripts

develop-image/docker-env.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Swap the following lines if you want to work with 'latest'
-DOCKER_IMAGE_TAG=valory/open-aea-develop:1.57.0
+DOCKER_IMAGE_TAG=valory/open-aea-develop:1.58.0
 # DOCKER_IMAGE_TAG=valory/open-aea-develop:latest
 
 DOCKER_BUILD_CONTEXT_DIR=..

docs/api/helpers/base.md

@@ -838,3 +838,13 @@ Prepend a path with a prefix, but only if not absolute
 
 the same path if absolute, else the prepended path.
 
+<a id="aea.helpers.base.update_nested_dict"></a>
+
+#### update`_`nested`_`dict
+
+```python
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict
+```
+
+Update a nested dictionary.
+