From e792de0f7ba32515345cf97fcdb6972131d3eeca Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 8 Jul 2024 14:29:19 +0530 Subject: [PATCH 1/3] Bumped up the version of certifi to 2024.7.4 --- .idea/databricks-sql-python.iml | 9 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 84 ++++++ poetry.lock | 8 +- pyproject.toml | 1 + .../sqlalchemy/test/requirements.py | 249 ++++++++++++++++++ src/databricks/sqlalchemy/test/setup.cfg | 4 + 9 files changed, 371 insertions(+), 4 deletions(-) create mode 100644 .idea/databricks-sql-python.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 src/databricks/sqlalchemy/test/requirements.py create mode 100644 src/databricks/sqlalchemy/test/setup.cfg diff --git a/.idea/databricks-sql-python.iml b/.idea/databricks-sql-python.iml new file mode 100644 index 00000000..1193e260 --- /dev/null +++ b/.idea/databricks-sql-python.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..cc6eae03 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..d3d7ab09 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..83f755d1 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + { + "associatedIndex": 4 +} + + + + + + + + + + + + + + + + + + + + 1719903395495 + + + + + + \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 86c05bba..a38d955c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,13 +72,13 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -1198,4 +1198,4 @@ sqlalchemy = ["sqlalchemy"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "d9fa00df9e0a90e8d44ee6bb88d291acd2f48c9c96d2a979fb06ad6e6de66529" +content-hash = "983b6405d99b6e4e931868c6eaf9136bcc64a3588d1a5db763358ee5753e4fdf" diff --git a/pyproject.toml b/pyproject.toml index 4320d4eb..401784c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ sqlalchemy = { version = ">=2.0.21", optional = true } openpyxl = "^3.0.10" alembic = { version = "^1.0.11", optional = true } urllib3 = ">=1.26" +certifi = "2024.7.4" [tool.poetry.extras] sqlalchemy = ["sqlalchemy"] diff --git a/src/databricks/sqlalchemy/test/requirements.py b/src/databricks/sqlalchemy/test/requirements.py new file mode 100644 index 00000000..75227efb --- /dev/null +++ b/src/databricks/sqlalchemy/test/requirements.py @@ -0,0 +1,249 @@ +""" +The complete list of requirements is provided by SQLAlchemy here: + +https://github.com/sqlalchemy/sqlalchemy/blob/main/lib/sqlalchemy/testing/requirements.py + +When SQLAlchemy skips a test because a requirement is closed() it gives a generic skip message. +To make these failures more actionable, we only define requirements in this file that we wish to +force to be open(). If a test should be skipped on Databricks, it will be specifically marked skip +in test_suite.py with a Databricks-specific reason. + +See the special note about the array_type exclusion below. +See special note about has_temp_table exclusion below. +""" + +import sqlalchemy.testing.requirements +import sqlalchemy.testing.exclusions + + +class Requirements(sqlalchemy.testing.requirements.SuiteRequirements): + @property + def date_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_literals(self): + """target dialect supports rendering of a date, time, or datetime as a + literal string, e.g. via the TypeEngine.literal_processor() method. + + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def timestamp_microseconds(self): + """target dialect supports representation of Python + datetime.datetime() with microsecond objects but only + if TIMESTAMP is used.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_microseconds(self): + """target dialect supports representation of Python + datetime.time() with microsecond objects. + + This requirement declaration isn't needed but I've included it here for completeness. + Since Databricks doesn't have a TIME type, SQLAlchemy will compile Time() columns + as STRING Databricks data types. And we use a custom time type to render those strings + between str() and time.time() representations. Therefore we can store _any_ precision + that SQLAlchemy needs. The time_microseconds requirement defaults to ON for all dialects + except mssql, mysql, mariadb, and oracle. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def infinity_floats(self): + """The Float type can persist and load float('inf'), float('-inf').""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_retains_significant_digits(self): + """A precision numeric type will return empty significant digits, + i.e. a value such as 10.000 will come back in Decimal form with + the .000 maintained.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_many_significant_digits(self): + """target backend supports values with many digits on both sides, + such as 319438950232418390.273596, 87673.594069654243 + + """ + return sqlalchemy.testing.exclusions.open() + + @property + def array_type(self): + """While Databricks does support ARRAY types, pysql cannot bind them. So + we cannot use them with SQLAlchemy + + Due to a bug in SQLAlchemy, we _must_ define this exclusion as closed() here or else the + test runner will crash the pytest process due to an AttributeError + """ + + # TODO: Implement array type using inline? + return sqlalchemy.testing.exclusions.closed() + + @property + def table_ddl_if_exists(self): + """target platform supports IF NOT EXISTS / IF EXISTS for tables.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY""" + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns_standard(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY with a standard syntax. + This is mainly to exclude MSSql. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def has_temp_table(self): + """target dialect supports checking a single temp table name + + unfortunately this is not the same as temp_table_names + + SQLAlchemy's HasTableTest is not normalised in such a way that temp table tests + are separate from temp view and normal table tests. If those tests were split out, + we would just add detailed skip markers in test_suite.py. But since we'd like to + run the HasTableTest group for the features we support, we must set this exclusinon + to closed(). + + It would be ideal if there were a separate requirement for has_temp_view. Without it, + we're in a bind. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def temporary_views(self): + """target database supports temporary views""" + return sqlalchemy.testing.exclusions.open() + + @property + def views(self): + """Target database must support VIEWs.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def temporary_tables(self): + """target database supports temporary tables + + ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def table_reflection(self): + """target database has general support for table reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection(self): + """Indicates if the database support table comment reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection_full_unicode(self): + """Indicates if the database support table comment reflection in the + full unicode range, including emoji etc. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def temp_table_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def index_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def unique_constraint_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + + Databricks doesn't support UNIQUE constraints. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def reflects_pk_names(self): + """Target driver reflects the name of primary key constraints.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_implicit_bound(self): + """target dialect when given a datetime object will bind it such + that the database server knows the object is a date, and not + a plain string. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def tuple_in(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes_with_update_delete(self): + return sqlalchemy.testing.exclusions.open() + + @property + def delete_from(self): + """Target must support DELETE FROM..FROM or DELETE..USING syntax""" + return sqlalchemy.testing.exclusions.open() + + @property + def table_value_constructor(self): + return sqlalchemy.testing.exclusions.open() + + @property + def reflect_tables_no_columns(self): + return sqlalchemy.testing.exclusions.open() + + @property + def denormalized_names(self): + """Target database must have 'denormalized', i.e. + UPPERCASE as case insensitive names.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_timezone(self): + """target dialect supports representation of Python + datetime.time() with tzinfo with Time(timezone=True).""" + + return sqlalchemy.testing.exclusions.open() diff --git a/src/databricks/sqlalchemy/test/setup.cfg b/src/databricks/sqlalchemy/test/setup.cfg new file mode 100644 index 00000000..ab89d17d --- /dev/null +++ b/src/databricks/sqlalchemy/test/setup.cfg @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks.sqlalchemy.requirements:Requirements +profile_file=profiles.txt From 3a3ebb1e8f1d74fcb11130405eb1aefa9136c2ce Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 8 Jul 2024 22:21:38 +0530 Subject: [PATCH 2/3] Removed uneccessary files --- .idea/databricks-sql-python.iml | 9 - .idea/misc.xml | 6 - .idea/modules.xml | 8 - .idea/vcs.xml | 6 - .idea/workspace.xml | 84 ------ .../sqlalchemy/test/requirements.py | 249 ------------------ src/databricks/sqlalchemy/test/setup.cfg | 4 - 7 files changed, 366 deletions(-) delete mode 100644 .idea/databricks-sql-python.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml delete mode 100644 src/databricks/sqlalchemy/test/requirements.py delete mode 100644 src/databricks/sqlalchemy/test/setup.cfg diff --git a/.idea/databricks-sql-python.iml b/.idea/databricks-sql-python.iml deleted file mode 100644 index 1193e260..00000000 --- a/.idea/databricks-sql-python.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index cc6eae03..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d3d7ab09..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 83f755d1..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - - - - - - - - - { - "associatedIndex": 4 -} - - - - - - - - - - - - - - - - - - - - 1719903395495 - - - - - - \ No newline at end of file diff --git a/src/databricks/sqlalchemy/test/requirements.py b/src/databricks/sqlalchemy/test/requirements.py deleted file mode 100644 index 75227efb..00000000 --- a/src/databricks/sqlalchemy/test/requirements.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -The complete list of requirements is provided by SQLAlchemy here: - -https://github.com/sqlalchemy/sqlalchemy/blob/main/lib/sqlalchemy/testing/requirements.py - -When SQLAlchemy skips a test because a requirement is closed() it gives a generic skip message. -To make these failures more actionable, we only define requirements in this file that we wish to -force to be open(). If a test should be skipped on Databricks, it will be specifically marked skip -in test_suite.py with a Databricks-specific reason. - -See the special note about the array_type exclusion below. -See special note about has_temp_table exclusion below. -""" - -import sqlalchemy.testing.requirements -import sqlalchemy.testing.exclusions - - -class Requirements(sqlalchemy.testing.requirements.SuiteRequirements): - @property - def date_historic(self): - """target dialect supports representation of Python - datetime.datetime() objects with historic (pre 1970) values.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def datetime_historic(self): - """target dialect supports representation of Python - datetime.datetime() objects with historic (pre 1970) values.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def datetime_literals(self): - """target dialect supports rendering of a date, time, or datetime as a - literal string, e.g. via the TypeEngine.literal_processor() method. - - """ - - return sqlalchemy.testing.exclusions.open() - - @property - def timestamp_microseconds(self): - """target dialect supports representation of Python - datetime.datetime() with microsecond objects but only - if TIMESTAMP is used.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def time_microseconds(self): - """target dialect supports representation of Python - datetime.time() with microsecond objects. - - This requirement declaration isn't needed but I've included it here for completeness. - Since Databricks doesn't have a TIME type, SQLAlchemy will compile Time() columns - as STRING Databricks data types. And we use a custom time type to render those strings - between str() and time.time() representations. Therefore we can store _any_ precision - that SQLAlchemy needs. The time_microseconds requirement defaults to ON for all dialects - except mssql, mysql, mariadb, and oracle. - """ - - return sqlalchemy.testing.exclusions.open() - - @property - def infinity_floats(self): - """The Float type can persist and load float('inf'), float('-inf').""" - - return sqlalchemy.testing.exclusions.open() - - @property - def precision_numerics_retains_significant_digits(self): - """A precision numeric type will return empty significant digits, - i.e. a value such as 10.000 will come back in Decimal form with - the .000 maintained.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def precision_numerics_many_significant_digits(self): - """target backend supports values with many digits on both sides, - such as 319438950232418390.273596, 87673.594069654243 - - """ - return sqlalchemy.testing.exclusions.open() - - @property - def array_type(self): - """While Databricks does support ARRAY types, pysql cannot bind them. So - we cannot use them with SQLAlchemy - - Due to a bug in SQLAlchemy, we _must_ define this exclusion as closed() here or else the - test runner will crash the pytest process due to an AttributeError - """ - - # TODO: Implement array type using inline? - return sqlalchemy.testing.exclusions.closed() - - @property - def table_ddl_if_exists(self): - """target platform supports IF NOT EXISTS / IF EXISTS for tables.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def identity_columns(self): - """If a backend supports GENERATED { ALWAYS | BY DEFAULT } - AS IDENTITY""" - return sqlalchemy.testing.exclusions.open() - - @property - def identity_columns_standard(self): - """If a backend supports GENERATED { ALWAYS | BY DEFAULT } - AS IDENTITY with a standard syntax. - This is mainly to exclude MSSql. - """ - return sqlalchemy.testing.exclusions.open() - - @property - def has_temp_table(self): - """target dialect supports checking a single temp table name - - unfortunately this is not the same as temp_table_names - - SQLAlchemy's HasTableTest is not normalised in such a way that temp table tests - are separate from temp view and normal table tests. If those tests were split out, - we would just add detailed skip markers in test_suite.py. But since we'd like to - run the HasTableTest group for the features we support, we must set this exclusinon - to closed(). - - It would be ideal if there were a separate requirement for has_temp_view. Without it, - we're in a bind. - """ - return sqlalchemy.testing.exclusions.closed() - - @property - def temporary_views(self): - """target database supports temporary views""" - return sqlalchemy.testing.exclusions.open() - - @property - def views(self): - """Target database must support VIEWs.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def temporary_tables(self): - """target database supports temporary tables - - ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. - This happens because we cannot skip individual combinations used in ComponentReflection test. - """ - return sqlalchemy.testing.exclusions.closed() - - @property - def table_reflection(self): - """target database has general support for table reflection""" - return sqlalchemy.testing.exclusions.open() - - @property - def comment_reflection(self): - """Indicates if the database support table comment reflection""" - return sqlalchemy.testing.exclusions.open() - - @property - def comment_reflection_full_unicode(self): - """Indicates if the database support table comment reflection in the - full unicode range, including emoji etc. - """ - return sqlalchemy.testing.exclusions.open() - - @property - def temp_table_reflection(self): - """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. - This happens because we cannot skip individual combinations used in ComponentReflection test. - """ - return sqlalchemy.testing.exclusions.closed() - - @property - def index_reflection(self): - """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. - This happens because we cannot skip individual combinations used in ComponentReflection test. - """ - return sqlalchemy.testing.exclusions.closed() - - @property - def unique_constraint_reflection(self): - """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. - This happens because we cannot skip individual combinations used in ComponentReflection test. - - Databricks doesn't support UNIQUE constraints. - """ - return sqlalchemy.testing.exclusions.closed() - - @property - def reflects_pk_names(self): - """Target driver reflects the name of primary key constraints.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def datetime_implicit_bound(self): - """target dialect when given a datetime object will bind it such - that the database server knows the object is a date, and not - a plain string. - """ - - return sqlalchemy.testing.exclusions.open() - - @property - def tuple_in(self): - return sqlalchemy.testing.exclusions.open() - - @property - def ctes(self): - return sqlalchemy.testing.exclusions.open() - - @property - def ctes_with_update_delete(self): - return sqlalchemy.testing.exclusions.open() - - @property - def delete_from(self): - """Target must support DELETE FROM..FROM or DELETE..USING syntax""" - return sqlalchemy.testing.exclusions.open() - - @property - def table_value_constructor(self): - return sqlalchemy.testing.exclusions.open() - - @property - def reflect_tables_no_columns(self): - return sqlalchemy.testing.exclusions.open() - - @property - def denormalized_names(self): - """Target database must have 'denormalized', i.e. - UPPERCASE as case insensitive names.""" - - return sqlalchemy.testing.exclusions.open() - - @property - def time_timezone(self): - """target dialect supports representation of Python - datetime.time() with tzinfo with Time(timezone=True).""" - - return sqlalchemy.testing.exclusions.open() diff --git a/src/databricks/sqlalchemy/test/setup.cfg b/src/databricks/sqlalchemy/test/setup.cfg deleted file mode 100644 index ab89d17d..00000000 --- a/src/databricks/sqlalchemy/test/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ - -[sqla_testing] -requirement_cls=databricks.sqlalchemy.requirements:Requirements -profile_file=profiles.txt From 8ac9b892b92e6fa60c6fb95d329e90ccc7cfea72 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 8 Jul 2024 22:29:02 +0530 Subject: [PATCH 3/3] Updated the poetry.lock files --- poetry.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3f99288f..2cad06cb 100755 --- a/poetry.lock +++ b/poetry.lock @@ -1202,5 +1202,4 @@ sqlalchemy = ["sqlalchemy"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "983b6405d99b6e4e931868c6eaf9136bcc64a3588d1a5db763358ee5753e4fdf" - +content-hash = "bb289639e9f2630940a6699c69905db0166d9ae50afb5485508f48bbf899c77c"