From af32d12d7464212958e86684dcb1c9318868cbb6 Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 27 Jan 2025 13:34:51 -0600 Subject: [PATCH 1/2] Add an optional table name postfix to the metadata builder --- python/felis/metadata.py | 4 +++- python/felis/tap_schema.py | 10 +++++++++- tests/test_metadata.py | 9 +++++++++ tests/test_tap_schema.py | 6 ++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/python/felis/metadata.py b/python/felis/metadata.py index 865a4300..ae73d361 100644 --- a/python/felis/metadata.py +++ b/python/felis/metadata.py @@ -134,6 +134,7 @@ def __init__( schema: Schema, apply_schema_to_metadata: bool = True, ignore_constraints: bool = False, + table_name_postfix: str = "", ) -> None: """Initialize the metadata builder.""" self.schema = schema @@ -142,6 +143,7 @@ def __init__( self.metadata = MetaData(schema=schema.name if apply_schema_to_metadata else None) self._objects: dict[str, Any] = {} self.ignore_constraints = ignore_constraints + self.table_name_postfix = table_name_postfix def build(self) -> MetaData: """Build the SQLAlchemy tables and constraints from the schema. @@ -225,7 +227,7 @@ def build_table(self, table_obj: datamodel.Table) -> None: description = table_obj.description columns = [self.build_column(column) for column in table_obj.columns] table = Table( - name, + name + self.table_name_postfix, self.metadata, *columns, comment=description, diff --git a/python/felis/tap_schema.py b/python/felis/tap_schema.py index e42fb084..df9dbd08 100644 --- a/python/felis/tap_schema.py +++ b/python/felis/tap_schema.py @@ -91,9 +91,15 @@ def __init__( self.table_name_postfix = table_name_postfix self.apply_schema_to_metadata = apply_schema_to_metadata self.schema_name = schema_name or TableManager._SCHEMA_NAME_STD + self.table_name_postfix = table_name_postfix if is_valid_engine(engine): assert isinstance(engine, Engine) + if table_name_postfix != "": + logger.warning( + "Table name postfix '%s' will be ignored when reflecting TAP_SCHEMA database", + table_name_postfix, + ) logger.debug( "Reflecting TAP_SCHEMA database from existing database at %s", engine.url._replace(password="***"), @@ -131,7 +137,9 @@ def _load_yaml(self) -> None: self.schema_name = self.schema.name self._metadata = MetaDataBuilder( - self.schema, apply_schema_to_metadata=self.apply_schema_to_metadata + self.schema, + apply_schema_to_metadata=self.apply_schema_to_metadata, + table_name_postfix=self.table_name_postfix, ).build() logger.debug("Loaded TAP_SCHEMA '%s' from YAML resource", self.schema_name) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 7939f0b3..e6c3a577 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -231,6 +231,15 @@ def test_ignore_constraints(self) -> None: msg=f"Table {table.name} has non-primary key constraints defined", ) + def test_table_name_postfix(self) -> None: + """Test that table name postfixes are correctly applied.""" + schema = Schema.model_validate(self.yaml_data) + schema.name = "main" + builder = MetaDataBuilder(schema, table_name_postfix="_test") + md = builder.build() + for table in md.tables.values(): + self.assertTrue(table.name.endswith("_test")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tap_schema.py b/tests/test_tap_schema.py index a6db6d3d..f9e04d19 100644 --- a/tests/test_tap_schema.py +++ b/tests/test_tap_schema.py @@ -60,6 +60,12 @@ def test_create_table_manager(self) -> None: # already been created. mgr = TableManager() + def test_table_name_postfix(self) -> None: + """Test the table name postfix.""" + mgr = TableManager(table_name_postfix="_test") + for table_name in mgr.metadata.tables: + self.assertTrue(table_name.endswith("_test")) + class DataLoaderTestCase(unittest.TestCase): """Test the `DataLoader` class.""" From b9ed31935dfe8399a27b183a4ce99f7f0f713701 Mon Sep 17 00:00:00 2001 From: Jeremy McCormick Date: Mon, 27 Jan 2025 15:43:20 -0600 Subject: [PATCH 2/2] Add missing semicolons when printing or writing SQL to file --- python/felis/tap_schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/felis/tap_schema.py b/python/felis/tap_schema.py index df9dbd08..368084ca 100644 --- a/python/felis/tap_schema.py +++ b/python/felis/tap_schema.py @@ -432,7 +432,7 @@ def load(self) -> None: # Execute the inserts if not in dry run mode. self._execute_inserts() else: - logger.info("Dry run: not loading data into database") + logger.info("Dry run - not loading data into database") def _insert_schemas(self) -> None: """Insert the schema data into the schemas table.""" @@ -573,7 +573,7 @@ def _compiled_inserts(self) -> list[str]: def _print_sql(self) -> None: """Print the generated inserts to stdout.""" for insert_str in self._compiled_inserts(): - print(insert_str) + print(insert_str + ";") def _write_sql_to_file(self) -> None: """Write the generated insert statements to a file.""" @@ -581,7 +581,7 @@ def _write_sql_to_file(self) -> None: raise ValueError("No output path specified") with open(self.output_path, "w") as outfile: for insert_str in self._compiled_inserts(): - outfile.write(insert_str + "\n") + outfile.write(insert_str + ";" + "\n") def _insert(self, table_name: str, record: list[Any] | dict[str, Any]) -> None: """Generate an insert statement for a record.