diff --git a/CHANGELOG.md b/CHANGELOG.md index c84527f..19bc248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Unreleased (TBD) +### Features +- Support dbt v1.4 ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) + +### Contributors +- [@lpezet](https://github.com/lpezet) ([#146](https://github.com/dbeatty10/dbt-mysql/pull/146)) + +## dbt-mysql 1.1.0 (Feb 5, 2023) + ### Features - Support dbt v1.1 ([#100](https://github.com/dbeatty10/dbt-mysql/pull/100)) - More clear exception for invalid `database` config ([#110](https://github.com/dbeatty10/dbt-mysql/issues/110), [#111](https://github.com/dbeatty10/dbt-mysql/pull/111)) diff --git a/dbt/adapters/mariadb/__version__.py b/dbt/adapters/mariadb/__version__.py index a6b9772..70ba273 100644 --- a/dbt/adapters/mariadb/__version__.py +++ b/dbt/adapters/mariadb/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mariadb/connections.py b/dbt/adapters/mariadb/connections.py index d85ec29..cd50ea9 100644 --- a/dbt/adapters/mariadb/connections.py +++ b/dbt/adapters/mariadb/connections.py @@ -40,11 +40,11 @@ def __init__(self, **kwargs): def __post_init__(self): # Database and schema are treated as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" - f"On MariaDB, database must be omitted or have the same value as" - f" schema." + f"On MariaDB, database must be omitted" + f" or have the same value as schema." ) @property @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,8 +164,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from + # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mariadb/impl.py b/dbt/adapters/mariadb/impl.py index 6a16bcb..2557f36 100644 --- a/dbt/adapters/mariadb/impl.py +++ b/dbt/adapters/mariadb/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MariaDBRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MariaDB database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mariadb__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -69,7 +71,8 @@ def list_relations_without_caching( return relations - def get_columns_in_relation(self, relation: Relation) -> List[MariaDBColumn]: + def get_columns_in_relation(self, + relation: Relation) -> List[MariaDBColumn]: rows: List[agate.Row] = super().get_columns_in_relation(relation) return self.parse_show_columns(relation, rows) @@ -89,7 +92,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -115,7 +118,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -144,7 +147,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mariadb _get_one_catalog, found " f"{schemas}" ) @@ -156,7 +159,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -199,7 +203,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) diff --git a/dbt/adapters/mariadb/relation.py b/dbt/adapters/mariadb/relation.py index 0b21aa0..fd1e883 100644 --- a/dbt/adapters/mariadb/relation.py +++ b/dbt/adapters/mariadb/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MariaDBIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MariaDBRelation(BaseRelation): - quote_policy: MariaDBQuotePolicy = MariaDBQuotePolicy() - include_policy: MariaDBIncludePolicy = MariaDBIncludePolicy() + quote_policy: MariaDBQuotePolicy = field( + default_factory=lambda: MariaDBQuotePolicy()) + include_policy: MariaDBIncludePolicy = field( + default_factory=lambda: MariaDBIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in MariaDB!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a MariaDB relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql/__version__.py b/dbt/adapters/mysql/__version__.py index a6b9772..70ba273 100644 --- a/dbt/adapters/mysql/__version__.py +++ b/dbt/adapters/mysql/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mysql/connections.py b/dbt/adapters/mysql/connections.py index 6a4e285..42880f6 100644 --- a/dbt/adapters/mysql/connections.py +++ b/dbt/adapters/mysql/connections.py @@ -39,7 +39,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -113,7 +113,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -138,19 +138,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -160,8 +160,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from the + # mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mysql/impl.py b/dbt/adapters/mysql/impl.py index f1e11d1..7e449ef 100644 --- a/dbt/adapters/mysql/impl.py +++ b/dbt/adapters/mysql/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mysql__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -89,7 +91,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -116,7 +118,7 @@ def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -145,7 +147,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mysql _get_one_catalog, found " f"{schemas}" ) @@ -157,7 +159,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -200,7 +203,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) @@ -227,7 +230,8 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, + # so we need to simulate it COLUMNS_EQUAL_SQL = """ WITH a_except_b as ( diff --git a/dbt/adapters/mysql/relation.py b/dbt/adapters/mysql/relation.py index 859afc1..5170dbd 100644 --- a/dbt/adapters/mysql/relation.py +++ b/dbt/adapters/mysql/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() - include_policy: MySQLIncludePolicy = MySQLIncludePolicy() + quote_policy: MySQLQuotePolicy = field( + default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field( + default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in mysql!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a mysql relation with schema and database set to " "include, but only one can be set" ) diff --git a/dbt/adapters/mysql5/__version__.py b/dbt/adapters/mysql5/__version__.py index a6b9772..70ba273 100644 --- a/dbt/adapters/mysql5/__version__.py +++ b/dbt/adapters/mysql5/__version__.py @@ -1 +1 @@ -version = "1.2.0a1" +version = "1.4.0a1" diff --git a/dbt/adapters/mysql5/connections.py b/dbt/adapters/mysql5/connections.py index c8c1d20..6199ff5 100644 --- a/dbt/adapters/mysql5/connections.py +++ b/dbt/adapters/mysql5/connections.py @@ -40,7 +40,7 @@ def __init__(self, **kwargs): def __post_init__(self): # mysql classifies database and schema as the same thing if self.database is not None and self.database != self.schema: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" f"On MySQL, database must be omitted or have the same value as" @@ -117,7 +117,7 @@ def open(cls, connection): connection.handle = None connection.state = "fail" - raise dbt.exceptions.FailedToConnectException(str(e)) + raise dbt.exceptions.FailedToConnectError(str(e)) return connection @@ -142,19 +142,19 @@ def exception_handler(self, sql): logger.debug("Failed to release connection!") pass - raise dbt.exceptions.DatabaseException(str(e).strip()) from e + raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e except Exception as e: logger.debug("Error running SQL: {}", sql) logger.debug("Rolling back transaction.") self.rollback_if_open() - if isinstance(e, dbt.exceptions.RuntimeException): + if isinstance(e, dbt.exceptions.DbtRuntimeError): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise - raise dbt.exceptions.RuntimeException(e) from e + raise dbt.exceptions.DbtRuntimeError(e) from e @classmethod def get_response(cls, cursor) -> AdapterResponse: @@ -164,8 +164,11 @@ def get_response(cls, cursor) -> AdapterResponse: if cursor is not None and cursor.rowcount is not None: num_rows = cursor.rowcount - # There's no real way to get the status from the mysql-connector-python driver. + # There's no real way to get the status from + # the mysql-connector-python driver. # So just return the default value. return AdapterResponse( - _message="{} {}".format(code, num_rows), rows_affected=num_rows, code=code + _message="{} {}".format(code, num_rows), + rows_affected=num_rows, + code=code ) diff --git a/dbt/adapters/mysql5/impl.py b/dbt/adapters/mysql5/impl.py index 2582c83..e0d61a3 100644 --- a/dbt/adapters/mysql5/impl.py +++ b/dbt/adapters/mysql5/impl.py @@ -32,7 +32,8 @@ def date_function(cls): return "current_date()" @classmethod - def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: + def convert_datetime_type(cls, agate_table: agate.Table, + col_idx: int) -> str: return "timestamp" def quote(self, identifier): @@ -43,8 +44,9 @@ def list_relations_without_caching( ) -> List[MySQLRelation]: kwargs = {"schema_relation": schema_relation} try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - except dbt.exceptions.RuntimeException as e: + results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, + kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") if f"MySQL database '{schema_relation}' not found" in errmsg: return [] @@ -56,7 +58,7 @@ def list_relations_without_caching( relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( "Invalid value from " f'"mysql5__list_relations_without_caching({kwargs})", ' f"got {len(row)} values, expected 4" @@ -89,7 +91,7 @@ def _get_columns_for_catalog( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: - if not self.Relation.include_policy.database: + if not self.Relation.get_default_include_policy().database: database = None return super().get_relation(database, schema, identifier) @@ -115,7 +117,7 @@ def parse_show_columns( def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) if len(schema_map) > 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" ) @@ -144,7 +146,7 @@ def _get_one_catalog( manifest, ) -> agate.Table: if len(schemas) != 1: - dbt.exceptions.raise_compiler_error( + raise dbt.exceptions.CompilationError( f"Expected only one schema in mysql5 _get_one_catalog, found " f"{schemas}" ) @@ -156,7 +158,8 @@ def _get_one_catalog( for relation in self.list_relations(database, schema): logger.debug("Getting table schema for relation {}", relation) columns.extend(self._get_columns_for_catalog(relation)) - return agate.Table.from_object(columns, column_types=DEFAULT_TYPE_TESTER) + return agate.Table.from_object(columns, + column_types=DEFAULT_TYPE_TESTER) def check_schema_exists(self, database, schema): results = self.execute_macro( @@ -199,7 +202,7 @@ def string_add_sql( elif location == "prepend": return f"concat({value}, '{add_to}')" else: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f'Got an unexpected location value of "{location}"' ) @@ -226,7 +229,8 @@ def get_rows_different_sql( ) first_column = names[0] - # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it + # MySQL doesn't have an EXCEPT or MINUS operator, + # so we need to simulate it COLUMNS_EQUAL_SQL = """ SELECT row_count_diff.difference as row_count_difference, diff --git a/dbt/adapters/mysql5/relation.py b/dbt/adapters/mysql5/relation.py index 1b03317..c7a150e 100644 --- a/dbt/adapters/mysql5/relation.py +++ b/dbt/adapters/mysql5/relation.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException +from dbt.exceptions import DbtRuntimeError @dataclass @@ -20,13 +20,15 @@ class MySQLIncludePolicy(Policy): @dataclass(frozen=True, eq=False, repr=False) class MySQLRelation(BaseRelation): - quote_policy: MySQLQuotePolicy = MySQLQuotePolicy() - include_policy: MySQLIncludePolicy = MySQLIncludePolicy() + quote_policy: MySQLQuotePolicy = field( + default_factory=lambda: MySQLQuotePolicy()) + include_policy: MySQLIncludePolicy = field( + default_factory=lambda: MySQLIncludePolicy()) quote_character: str = "`" def __post_init__(self): if self.database != self.schema and self.database: - raise RuntimeException( + raise DbtRuntimeError( f"Cannot set `database` to '{self.database}' in mysql5!" "You can either unset `database`, or make it match `schema`, " f"currently set to '{self.schema}'" @@ -34,7 +36,7 @@ def __post_init__(self): def render(self): if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( + raise DbtRuntimeError( "Got a mysql5 relation with schema and database set to " "include, but only one can be set" ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7e63b22..4c54314 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? # git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@1.2.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@1.4.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter wheel twine