diff --git a/python/lib/core/dmod/core/common/collection.py b/python/lib/core/dmod/core/common/collection.py index dcdd65a25..860f8083a 100644 --- a/python/lib/core/dmod/core/common/collection.py +++ b/python/lib/core/dmod/core/common/collection.py @@ -492,5 +492,5 @@ def _inner_sequence(self) -> typing.MutableSequence[_T]: def _get_handlers(self) -> typing.Dict[CollectionEvent, typing.MutableSequence[typing.Callable]]: return self._handlers - __root__: list[_T] = pydantic.Field(default_factory=list) + __root__: typing.List[_T] = pydantic.Field(default_factory=list) _handlers: typing.Dict[CollectionEvent, typing.List[typing.Callable]] = PrivateAttr(default_factory=dict) \ No newline at end of file diff --git a/python/lib/core/dmod/core/common/helper_functions.py b/python/lib/core/dmod/core/common/helper_functions.py index 75c38b687..afcf18e0f 100644 --- a/python/lib/core/dmod/core/common/helper_functions.py +++ b/python/lib/core/dmod/core/common/helper_functions.py @@ -662,6 +662,15 @@ def flat(collection: typing.Iterable[typing.Iterable[_CLASS_TYPE]]) -> typing.Se >>> example_values = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] >>> flat(example_values) [1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> second_example_values = { + ... "one": [1, 2, 3], + ... "two": [4, 5, 6], + ... "three": 7, + ... "four": 8, + ... "five": 9 + ... } + >>> flat(second_example_values) + [1, 2, 3, 4, 5, 6, 7, 8, 9] Args: collection: The collection of collections to flatten @@ -671,8 +680,15 @@ def flat(collection: typing.Iterable[typing.Iterable[_CLASS_TYPE]]) -> typing.Se """ flattened_list: typing.MutableSequence[_CLASS_TYPE] = list() - for inner_collection in collection: - flattened_list.extend(inner_collection) + if isinstance(collection, typing.Mapping): + for mapped_value in collection.values(): + if is_iterable_type(mapped_value): + flattened_list.extend(mapped_value) + else: + flattened_list.append(mapped_value) + else: + for inner_collection in collection: + flattened_list.extend(inner_collection) return flattened_list diff --git a/python/lib/evaluations/dmod/evaluations/specification/template.py b/python/lib/evaluations/dmod/evaluations/specification/template.py index ab4d17e54..dc1828561 100644 --- a/python/lib/evaluations/dmod/evaluations/specification/template.py +++ b/python/lib/evaluations/dmod/evaluations/specification/template.py @@ -143,7 +143,7 @@ def initialize(self, connection: DBAPIConnection): if cursor: cursor.close() - def create_keyed_insert_query(self, columns: typing.Sequence[str]) -> QueryDetails: + def create_keyed_insert_query(self) -> QueryDetails: """ Generate a query that will insert unique values into the table @@ -153,28 +153,20 @@ def create_keyed_insert_query(self, columns: typing.Sequence[str]) -> QueryDetai Returns: A sql query that will insert data into a database table """ - missing_columns: typing.Set[str] = self.required_columns.union(self.keys).difference(columns) - - if missing_columns: - raise ValueError( - f"Cannot create a script to insert values into '{self.name}' - " - f"missing the following required columns: {', '.join(columns)}" - ) - # Insert values into each column that don't already exist based on the given values query = f""" INSERT INTO {self.name} - ({', '.join(columns)}) -SELECT {', '.join(['?' for _ in columns])} + ({', '.join(self.column_names)}) +SELECT {', '.join(['?' for _ in self.column_names])} WHERE NOT EXISTS ( SELECT 1 FROM {self.name} WHERE {' AND '.join([f'{key} = ?' for key in self.keys])} )""" - labels = [name for name in columns] + [key for key in self.keys] + labels = [name for name in self.column_names] + [key for key in self.keys] return QueryDetails(query=query, value_labels=labels) - def create_unkeyed_insert_query(self, columns: typing.Sequence[str]) -> QueryDetails: + def create_unkeyed_insert_query(self) -> QueryDetails: """ Create an insert query for this table that does not identify values based on keys @@ -184,21 +176,13 @@ def create_unkeyed_insert_query(self, columns: typing.Sequence[str]) -> QueryDet Returns: An insert script for this table that will insert values into the given columns """ - missing_columns: typing.Set[str] = self.required_columns.difference(columns) - - if missing_columns: - raise ValueError( - f"Cannot create a script to insert values into '{self.name}' - " - f"missing the following required columns: {', '.join(columns)}" - ) - # Insert values into the given database query = f""" INSERT INTO {self.name} - ({', '.join(columns)} -VALUES ({', '.join(['?' for _ in columns])}); + ({', '.join(self.column_names)}) +VALUES ({', '.join(['?' for _ in self.column_names])}); """ - labels = [name for name in columns] + labels = [name for name in self.column_names] return QueryDetails(query=query, value_labels=labels) def override_table(self, connection: DBAPIConnection): @@ -219,7 +203,7 @@ def override_table(self, connection: DBAPIConnection): try: cursor = connection.cursor() - cursor.execute(f"TRUNCATE TABLE {self.name}") + cursor.execute(f"DROP TABLE {self.name}") cursor.fetchall() try: @@ -258,24 +242,41 @@ def insert_templates( query_details: QueryDetails if self.keys and not override: - query_details = self.create_keyed_insert_query(self.column_names) + query_details = self.create_keyed_insert_query() else: - query_details = self.create_unkeyed_insert_query(self.column_names) + query_details = self.create_unkeyed_insert_query() query: str = query_details.query - parameters: typing.Sequence[typing.Sequence[typing.Any]] = [ - [ - getattr(row, label) - for label in query_details.value_labels - ] - for row in rows - ] + parameters: typing.List[typing.Sequence[typing.Any]] = list() + + for template in rows: + template_parameters: typing.List[str, int, float, bool] = list() + + for label in query_details.value_labels: + # The configuration may not always be on the template object and will need to be read, so call + # 'get_configuration' instead of just grabbing 'configuration' + if label == 'configuration': + template_parameters.append( + json.dumps( + template.get_configuration() + ) + ) + else: + template_parameters.append(getattr(template, label)) + + parameters.append(template_parameters) cursor: typing.Optional[DBAPICursor] = None + try: cursor = connection.cursor() - cursor.executemany(query, parameters) + + try: + cursor.executemany(query, parameters) + except: + print(query, file=sys.stderr) + raise try: connection.commit() @@ -308,14 +309,12 @@ def get_template_table(name: str) -> Table: Column(name="name", datatype="VARCHAR(255)"), Column(name="specification_type", datatype="VARCHAR(255)"), Column(name="description", datatype="VARCHAR(500)", optional=True), - Column(name="author", datatype="VARCHAR(500)", optional=True), + Column(name="author_name", datatype="VARCHAR(500)", optional=True), Column(name="configuration", datatype="TEXT"), Column(name="last_modified", datatype="VARCHAR(50)") ], keys=[ - 'name', - 'specification_type', - 'author' + 'name' ] ) @@ -511,6 +510,12 @@ def __eq__(self, other): and self.specification_type == other.specification_type \ and self.get_configuration() == other.get_configuration() + def __str__(self): + return f"[{self.specification_type}] {self.name}{f' : {self.description}' if self.description else ''}" + + def __repr__(self): + return self.__str__() + class TemplateManager(abc.ABC, TemplateManagerProtocol): """ diff --git a/python/lib/evaluations/dmod/evaluations/specification/templates/file_templates.py b/python/lib/evaluations/dmod/evaluations/specification/templates/file_templates.py index 80dec7e77..b3deb04ce 100644 --- a/python/lib/evaluations/dmod/evaluations/specification/templates/file_templates.py +++ b/python/lib/evaluations/dmod/evaluations/specification/templates/file_templates.py @@ -87,6 +87,12 @@ def dict(self) -> typing.Dict[str, typing.Union[str, pathlib.Path]]: return details + def __str__(self): + return f"[{self.specification_type}] {self.name}{f' : {self.description}' if self.description else ''}" + + def __repr__(self): + return self.__str__() + def serialize_path(path: pathlib.Path, *args, **kwargs) -> str: return str(path) diff --git a/python/services/evaluationservice/dmod/evaluationservice/evaluation_service/models.py b/python/services/evaluationservice/dmod/evaluationservice/evaluation_service/models.py index f90c14a87..d30af03d3 100644 --- a/python/services/evaluationservice/dmod/evaluationservice/evaluation_service/models.py +++ b/python/services/evaluationservice/dmod/evaluationservice/evaluation_service/models.py @@ -76,11 +76,14 @@ def field_choice(self) -> typing.Tuple[str, str]: return self.name, self.name def to_details(self) -> TemplateDetails: - return BasicTemplateDetails.from_details(self) + return BasicTemplateDetails.copy(self) def __str__(self): return f"[{self.specification_type}] {self.name}{':' + self.description if self.description else ''}" + def __repr__(self): + return self.__str__() + class EvaluationDefinition(models.Model): """