Update to 0.1.62 from 'dev' branch

AoE2ScenarioParser/helper/bytes_conversions.py

@@ -100,14 +100,14 @@ def bytes_to_str(
             continue
 
     # Return the string as bytes when it cannot be decoded. This will leave it as-is.
-    truncated = trunc_bytes(byte_elements, 25)
+    truncated = trunc_string(byte_elements, 25)
     warn(f"Unable to decode bytes using '{charset}' and '{fallback_charset}', bytes: \n\t{truncated}",
          category=ByteDecodeWarning)
     return byte_elements
 
 
 def str_to_bytes(
-        string: str,
+        string: str | bytes,
         charset: str = settings.MAIN_CHARSET,
         fallback_charset: str = settings.FALLBACK_CHARSET
 ) -> bytes:
@@ -130,6 +130,10 @@ def str_to_bytes(
             return string.encode(c)
         except ValueError:
             continue
+        except AttributeError as e:
+            if type(string) is bytes:
+                return string
+            raise e
     raise ValueError(f"Unable to encode string using '{charset}' and '{fallback_charset}'. String:\n\t{q_str(string)}")
 
 
@@ -326,6 +330,6 @@ def _combine_int_str(
         if str(e) == "int too big to convert":
             name: str = retriever.name if retriever else ""
             raise OverflowError(
-                f"{name.capitalize()} is too long. "
+                f"'{name.capitalize()}' is too long. "
                 f"Length must be below {pow(2, length * 8 - 1)}, but is: {len(byte_string)}"
             ) from None

AoE2ScenarioParser/helper/string_manipulations.py

@@ -119,7 +119,7 @@ def trunc_string(string: str | bytes, length=30, add_ellipsis=True) -> str | byt
     Truncates string to a certain length and possibly adds ellipsis to said string.
 
     Args:
-        string: The string to truncate if over the given length
+        string: The string to truncate if its length is longer than the specified max (see length)
         length: The max given length the string can be
         add_ellipsis: If ellipsis should be added if the string is truncated
 
@@ -142,8 +142,8 @@ def add_tabs(string: str, tabs: int) -> str:
     Returns:
         The tabbed string
     """
-    splitted_string = string.splitlines(keepends=True)
-    return ("\t" * tabs) + ("\t" * tabs).join(splitted_string)
+    split_string = string.splitlines(keepends=True)
+    return ("\t" * tabs) + ("\t" * tabs).join(split_string) 
 
 
 def create_inline_line(entries: List[Any]):

AoE2ScenarioParser/objects/data_objects/player/player.py

@@ -46,6 +46,7 @@ def __init__(
             player_id: int,
             starting_age: int,
             lock_civ: int,
+            lock_personality: int,
             food: int,
             wood: int,
             gold: int,
@@ -76,6 +77,7 @@ def __init__(
         self._active: bool = active
         self.starting_age: int = dataset_or_value(StartingAge, starting_age)
         self.lock_civ: bool = bool(lock_civ)
+        self.lock_personality: bool = bool(lock_personality)
         self.food: int = food
         self.wood: int = wood
         self.gold: int = gold

AoE2ScenarioParser/objects/managers/player_manager.py

@@ -33,6 +33,7 @@ class PlayerManager(AoE2Object):
             RetrieverObjectLink("_string_table_player_names", link="string_table_player_names"),
             RetrieverObjectLink("_metadata", link="player_data_1", process_as_object=PlayerMetaData),
             RetrieverObjectLink("_lock_civilizations", link="per_player_lock_civilization"),
+            RetrieverObjectLink("_lock_personalities", link="per_player_lock_personality"),
         ]),
 
         RetrieverObjectLink("_resources", "PlayerDataTwo", "resources", process_as_object=PlayerResources),
@@ -66,6 +67,7 @@ def __init__(
             _string_table_player_names: List[int],
             _metadata: List[PlayerMetaData],
             _lock_civilizations: List[int],
+            _lock_personalities: List[int],
             _resources: List[PlayerResources],
             _diplomacy: List[PlayerDiplomacy],
             _allied_victories: List[int],
@@ -99,6 +101,7 @@ def __init__(
         gaia_last_params = {
             'starting_age': _starting_ages,
             'lock_civ': _lock_civilizations,
+            'lock_personality': _lock_personalities,
             'food': [r.food for r in _resources],
             'wood': [r.wood for r in _resources],
             'gold': [r.gold for r in _resources],
@@ -212,6 +215,11 @@ def _lock_civilizations(self):
         """Returns the civ lock bool of all players"""
         return self._player_attributes_to_list("lock_civ", False, default=0, fill_empty=7)
 
+    @property
+    def _lock_personalities(self):
+        """Returns the civ lock bool of all players"""
+        return self._player_attributes_to_list("lock_personality", False, default=0, fill_empty=7)
+
     @property
     def _pop_caps(self):
         """Returns the population cap of all players"""
@@ -291,7 +299,9 @@ def _resources(self) -> List[PlayerResources]:
         wood = self._player_attributes_to_list("wood", False, default=0, fill_empty=7)
         gold = self._player_attributes_to_list("gold", False, default=0, fill_empty=7)
         stone = self._player_attributes_to_list("stone", False, default=0, fill_empty=7)
-        color = self._player_attributes_to_list("color", False, default=0, fill_empty=7)
+        color = self._player_attributes_to_list("color", False, default=0)
+        color.extend(range(9, 16))
+
         return UuidList(self._uuid, [
             PlayerResources(food[i], wood[i], gold[i], stone[i], color[i]) for i in range(len(food))
         ])
@@ -300,7 +310,7 @@ def _resources(self) -> List[PlayerResources]:
     def _metadata(self) -> List[PlayerMetaData]:
         """Returns the metadata objects for all players"""
         active = self._player_attributes_to_list("active", False, default=0, fill_empty=7)
-        human = self._player_attributes_to_list("human", False, default=0, fill_empty=7)
+        human = self._player_attributes_to_list("human", False, default=1, fill_empty=7)
         civilization = self._player_attributes_to_list("civilization", False, default=Civilization.RANDOM, fill_empty=7)
         architecture_set = self._player_attributes_to_list("architecture_set", False, default=Civilization.RANDOM, fill_empty=7)
         return UuidList(self._uuid, [
@@ -320,7 +330,7 @@ def _tribe_names(self) -> List[str]:
     @property
     def _string_table_player_names(self) -> List[int]:
         """Returns the string table player names of all players"""
-        return self._player_attributes_to_list("string_table_name_id", None, default=4294967294, fill_empty=8)
+        return self._player_attributes_to_list("string_table_name_id", None, default=-2, fill_empty=8)
 
     def _player_attributes_to_list(
             self,

AoE2ScenarioParser/sections/aoe2_file_section.py

@@ -6,12 +6,12 @@
 
 from AoE2ScenarioParser.helper import bytes_parser
 from AoE2ScenarioParser.helper.list_functions import listify
-from AoE2ScenarioParser.helper.string_manipulations import create_textual_hex, insert_char, add_suffix_chars, q_str
+from AoE2ScenarioParser.helper.string_manipulations import create_textual_hex, insert_char, add_suffix_chars, q_str, \
+    trunc_string
 from AoE2ScenarioParser.sections.aoe2_struct_model import AoE2StructModel, model_dict_from_structure
 from AoE2ScenarioParser.sections.dependencies.dependency import handle_retriever_dependency
 from AoE2ScenarioParser.sections.retrievers.retriever import Retriever, reset_retriever_map
 
-
 if TYPE_CHECKING:
     from AoE2ScenarioParser.helper.incremental_generator import IncrementalGenerator
 
@@ -47,13 +47,15 @@ class AoE2FileSection:
     Utilises the AoE2StructModel to create multiple of the same structure. These copies are what is used to store the
     data from a scenario file
     """
-    def __init__(self,
-                 name: str,
-                 retriever_map: Dict[str, 'Retriever'],
-                 uuid: UUID,
-                 struct_models: Dict[str, 'AoE2StructModel'] = None,
-                 level: SectionLevel = SectionLevel.TOP_LEVEL
-                 ):
+
+    def __init__(
+            self,
+            name: str,
+            retriever_map: Dict[str, 'Retriever'],
+            uuid: UUID,
+            struct_models: Dict[str, 'AoE2StructModel'] = None,
+            level: SectionLevel = SectionLevel.TOP_LEVEL
+    ):
         """
         Args:
             name: The name of the file section
@@ -201,7 +203,7 @@ def _fill_retriever_with_bytes(self, retriever: 'Retriever', retrieved_bytes: Li
             print("\n\n" + "#" * 120)
             print(f"[{e.__class__.__name__}] Occurred while setting data in:\n\t{retriever}")
             print("#" * 120)
-            print(self.get_byte_structure_as_string())
+            print(trunc_string(self.get_byte_structure_as_string(), length=10000))
             raise e
 
     def _create_struct(self, model: 'AoE2StructModel', igenerator: 'IncrementalGenerator') -> 'AoE2FileSection':
@@ -227,29 +229,11 @@ def _create_struct(self, model: 'AoE2StructModel', igenerator: 'IncrementalGener
             print(f"[{e.__class__.__name__}] Occurred while setting data in: {struct.name}"
                   f"\n\tContent of {self.name}:")
             print("#" * 120)
-            print(self.get_byte_structure_as_string())
+            print(trunc_string(self.get_byte_structure_as_string(), length=10000))
             raise e
 
         return struct
 
-    # Todo: Remove if unused
-    # def set_data(self, data):
-    #     retrievers = list(self.retriever_map.values())
-    #
-    #     if len(data) == len(retrievers):
-    #         for i in range(len(data)):
-    #             retrievers[i].data = data[i]
-    #
-    #             if hasattr(retrievers[i], 'on_construct'):
-    #                 handle_retriever_dependency(retrievers[i], "construct", self, self._uuid)
-    #     else:
-    #         print(f"\nError in: {self.__class__.__name__}")
-    #         print(f"Data: (len: {len(data)}) "
-    #               f"{pretty_format_list([f'{i}: {str(x)}' for i, x in enumerate(data)])}")
-    #         print(f"Retrievers: (len: {len(retrievers)}) "
-    #               f"{pretty_format_list([f'{i}: {str(x)}' for i, x in enumerate(self.retriever_map.values())])}")
-    #         raise ValueError("Data list isn't the same size as the DataType list")
-
     def __getattr__(self, item):
         """Providing a default way to access retriever data labeled 'name'"""
         if 'retriever_map' not in self.__dict__:

AoE2ScenarioParser/sections/aoe2_struct_model.py

@@ -42,7 +42,10 @@ def from_structure(cls, name: str, structure: Dict[str, Dict]) -> AoE2StructMode
             An AoE2StructModel instance representing the given structure
         """
         retriever_map = {}
+        log_all_retrievers = structure.get('log', False)
         for retriever_name, attr in structure.get('retrievers').items():
+            if log_all_retrievers:
+                attr['log'] = True
             retriever_map[retriever_name] = Retriever.from_structure(retriever_name, attr)
         structs = model_dict_from_structure(structure)
 

AoE2ScenarioParser/sections/retrievers/datatype.py

@@ -62,9 +62,9 @@ def __init__(self, var: str = "0", repeat: int = 1, log_value: bool = False, typ
             type_length (Default: None):
                 A tuple specifying the type and length of a retriever
         """
-        self.var = var
-        self._repeat = repeat
         self.log_value = log_value
+        self.var = var
+        self.repeat = repeat
 
         if type_length is not None:
             self.type, self.length = type_length
@@ -103,7 +103,10 @@ def repeat(self, value: int):
         Args:
             value: The value to set repeat to. The value of an assignment is automatically passed as this argument
         """
-        if self.log_value:
+        if type(value) is not int:
+            raise TypeError(f"Repeat value must be an integer, now: {value}")
+
+        if self.log_value and hasattr(self, '_debug_retriever_name'):
             print(f"[DataType] {self._debug_retriever_name} 'repeat' set to {value} (was: {self._repeat})")
         self._repeat = value
 
@@ -131,7 +134,7 @@ def duplicate(self) -> DataType:
         )
 
     def __repr__(self):
-        return f"[DataType] " + self.to_simple_string()
+        return f"[DataType]  " + self.to_simple_string()
 
 
 def datatype_to_type_length(var: str) -> tuple:

AoE2ScenarioParser/sections/retrievers/retriever.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pickle
+from time import sleep
 from typing import Dict, List, Any
 
 from AoE2ScenarioParser import settings
@@ -9,7 +10,7 @@
 from AoE2ScenarioParser.helper.bytes_conversions import parse_bytes_to_val, parse_val_to_bytes
 from AoE2ScenarioParser.helper.list_functions import listify
 from AoE2ScenarioParser.helper.pretty_format import pretty_format_list
-from AoE2ScenarioParser.helper.printers import warn
+from AoE2ScenarioParser.helper.printers import warn, s_print
 from AoE2ScenarioParser.sections.dependencies.dependency_action import DependencyAction
 from AoE2ScenarioParser.sections.dependencies.retriever_dependency import RetrieverDependency
 from AoE2ScenarioParser.sections.retrievers.datatype import DataType
@@ -46,6 +47,7 @@ def __init__(
             datatype: DataType = DataType(),
             is_list: bool | None = None,
             log_value: bool = False,
+            data: Any = None
     ):
         """
         Args:
@@ -63,7 +65,7 @@ def __init__(
         self.is_list: bool = is_list
         self.log_value: bool = log_value
         self.is_dirty: bool = False
-        self._data: Any = None
+        self._data: Any = data
 
         if log_value:
             self.datatype.log_value = True
@@ -81,11 +83,20 @@ def get_data_as_bytes(self) -> bytes:
         if self.data is not None and self.datatype.repeat != 0:
             result = []
             if self.datatype.type == "struct":
-                for struct in self.data:
-                    result.append(struct.get_data_as_bytes())
+                if isinstance(self.data, list):
+                    for struct in self.data:
+                        result.append(struct.get_data_as_bytes())
+                else:
+                    result.append(self.data.get_data_as_bytes())
             else:
-                for value in listify(self.data):
-                    result.append(parse_val_to_bytes(self, value))
+                try:
+                    for value in listify(self.data):
+                        result.append(parse_val_to_bytes(self, value))
+                except Exception as e:
+                    print("\nException occurred during non-struct list creation: ")
+                    print(self)
+                    print(pretty_format_list(result))
+                    raise e
 
             joined_result = b''.join(result)
         else:
@@ -110,14 +121,14 @@ def set_data_from_bytes(self, bytes_list: List[bytes]) -> None:
         if self.datatype.repeat > 0 and len(bytes_list) == 0:
             raise ValueError("Unable to set bytes when no bytes are given")
         if self.datatype.repeat > 0 and self.datatype.repeat != len(bytes_list):
-            raise ValueError("Unable to set bytes when bytes list isn't equal to repeat")
+            raise ValueError(f"Unable to set bytes when bytes list isn't equal to repeat (repeat: {self.datatype.repeat} vs list: {len(bytes_list)})")
 
         result = [parse_bytes_to_val(self, entry_bytes) for entry_bytes in bytes_list]
         self.data = bytes_parser.vorl(self, result)
 
     def update_datatype_repeat(self) -> None:
         """Sets all the datatype repeat values to the lengths of their containing lists"""
-        if type(self.data) == list:
+        if type(self.data) is list:
             self.datatype.repeat = len(self.data)
 
     @property
@@ -219,6 +230,7 @@ def from_structure(cls, name: str, structure: dict) -> Retriever:
                     _evaluate_is_list_attribute(retriever, r_dep)
                     dependency_list.append(r_dep)
                 setattr(retriever, dependency_name, dependency_list)
+
         return retriever
 
     def print_value_update(self, old: Any, new: Any) -> None:

AoE2ScenarioParser/versions/DE/v1.36/structure.json

@@ -119,7 +119,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,

AoE2ScenarioParser/versions/DE/v1.37/structure.json

@@ -117,7 +117,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,

AoE2ScenarioParser/versions/DE/v1.40/structure.json

@@ -113,7 +113,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,

AoE2ScenarioParser/versions/DE/v1.41/structure.json

@@ -113,7 +113,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,

AoE2ScenarioParser/versions/DE/v1.42/structure.json

@@ -113,7 +113,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,

AoE2ScenarioParser/versions/DE/v1.43/structure.json

@@ -113,7 +113,7 @@
                 ]
             },
             "string_table_player_names": {
-                "type": "u32",
+                "type": "s32",
                 "repeat": 16,
                 "default": [
                     4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294, 4294967294,