Merge pull request #27 from dssg/migrate-sqlalchemy-1.4

Migrated to  sqlalchemy 1.4

README.rst

@@ -803,7 +803,7 @@ of 896,677 rows across 83 columns: 1 of these of type timestamp, 51
 integers and 31 floats. The benchmarking package, ``prof``, is
 preserved in `Ohio's repository <https://github.com/dssg/ohio>`_.
 
-.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true
+.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true
 
 ohio_pg_copy_from_X
    ``pg_copy_from(buffer_size=X)``
@@ -831,7 +831,7 @@ pandas_read_csv_stringio
    ``COPY`` to a ``StringIO``, from which pandas constructs a
    ``DataFrame``.
 
-.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true
+.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true
 
 ohio_pg_copy_to
    ``pg_copy_to()``

doc/conf.py

@@ -23,7 +23,7 @@
 author = 'Jesse London'
 
 # The full version, including alpha/beta/rc tags
-release = '0.5.0'
+release = '0.6.0'
 
 
 # -- General configuration ---------------------------------------------------

doc/index.rst

@@ -113,7 +113,7 @@ Modules
     .. Also note the query string ?sanitize=true
        This is necessary to ensure that SVG is returned with the image content type.
 
-    .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true
+    .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true
 
     ohio_pg_copy_from_X
         ``pg_copy_from(buffer_size=X)``
@@ -141,7 +141,7 @@ Modules
         ``COPY`` to a ``StringIO``, from which pandas constructs a
         ``DataFrame``.
 
-    .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true
+    .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true
 
     ohio_pg_copy_to
         ``pg_copy_to()``

requirement/dev.txt

@@ -1,5 +1,5 @@
 argcmdr==0.6.0
-bump2version==0.5.10
-tox==3.7.0
-twine==1.13.0
-wheel==0.33.1
+bump2version==1.0.1
+tox==3.20.0
+twine==1.15.0
+wheel==0.37.0

requirement/docs.txt

@@ -1,3 +1,3 @@
 -r ./include/ext.txt
 https://github.com/dssg/restbuilder/archive/jsl.zip
-sphinx==2.0.0
+sphinx==4.5.0

requirement/include/ext.txt

@@ -1,2 +1,2 @@
-numpy==1.16.2
-pandas==0.24.1
+numpy==1.21.6
+pandas==1.4.2

requirement/profile.txt

@@ -1,5 +1,5 @@
 -r ./include/ext.txt
-matplotlib==3.0.3
+matplotlib==3.5.2
 memory-profiler==0.55.0
-sqlalchemy==1.3.1
+sqlalchemy==1.4.35
 testing.postgresql==1.3.0

requirement/test.txt

@@ -1,6 +1,6 @@
 -r ./include/ext.txt
-psycopg2-binary==2.7.7
-pytest==4.3.0
+psycopg2-binary==2.9.3
+pytest==6.2.5
 signalled-timeout==1.0.0
-sqlalchemy==1.3.1
+sqlalchemy==1.4.35
 testing.postgresql==1.3.0

setup.cfg

@@ -2,7 +2,7 @@
 commit = True
 tag = True
 tag_name = {new_version}
-current_version = 0.5.0
+current_version = 0.6.0
 
 [bumpversion:file:setup.py]
 
@@ -19,4 +19,3 @@ test = pytest
 
 [metadata]
 license_files = LICENSE
-

setup.py

@@ -31,7 +31,7 @@ def stream_requirements(fd):
 
 setup(
     name='ohio',
-    version='0.5.0',
+    version='0.6.0',
     description="I/O extras",
     long_description=README_PATH.read_text(),
     long_description_content_type='text/x-rst',

src/ohio/__init__.py

@@ -35,4 +35,4 @@
 )
 
 
-__version__ = '0.5.0'
+__version__ = '0.6.0'

src/ohio/ext/pandas.py

@@ -60,6 +60,7 @@ class DataFramePgCopyTo:
     (excepting parameter ``method``).
 
     """
+
     def __init__(self, data_frame):
         self.data_frame = data_frame
 
@@ -72,7 +73,7 @@ def __call__(self, *args, **kwargs):
         )
 
 
-def to_sql_method_pg_copy_to(table, conn, keys, data_iter):
+def to_sql_method_pg_copy_to(table, engine, keys, data_iter):
     """Write pandas data to table via stream through PostgreSQL
     ``COPY``.
 
@@ -91,12 +92,13 @@ def to_sql_method_pg_copy_to(table, conn, keys, data_iter):
         columns=columns,
     )
 
-    with ohio.CsvTextIO(data_iter) as csv_buffer, \
-            conn.connection.cursor() as cursor:
-        cursor.copy_expert(sql, csv_buffer)
+    with ohio.CsvTextIO(data_iter) as csv_buffer:
+        with engine.connect() as conn:
+            with conn.connection.cursor() as cursor:
+                cursor.copy_expert(sql, csv_buffer)
 
 
-def data_frame_pg_copy_from(sql, connectable,
+def data_frame_pg_copy_from(sql, engine,
                             schema=None, index_col=None, parse_dates=False,
                             columns=None, dtype=None, nrows=None,
                             buffer_size=100):
@@ -131,7 +133,7 @@ def data_frame_pg_copy_from(sql, connectable,
     ``100``.
 
     """
-    pandas_sql = pandas.io.sql.SQLDatabase(connectable)
+    pandas_sql = pandas.io.sql.SQLDatabase(engine)
 
     try:
         is_table_name = pandas_sql.has_table(sql, schema)
@@ -158,30 +160,32 @@ def data_frame_pg_copy_from(sql, connectable,
             else:
                 schema_description = 'default schema'
 
-            hint = " (hint: no such table {!r} found under {})".format(sql, schema_description)
+            hint = " (hint: no such table {!r} found under {})".format(
+                sql, schema_description)
 
         raise TypeError("keyword arguments 'columns' and 'schema' supported "
                         "only when copying from table" + hint)
     else:
         source = "({})".format(sql)
 
-    with connectable.connect() as conn:
-        cursor = conn.connection.cursor()
-
-        writer = functools.partial(
-            cursor.copy_expert,
-            'COPY {source} TO STDOUT WITH CSV HEADER'.format(source=source),
-        )
+    with engine.connect() as conn:
+        with conn.connection.cursor() as cursor:
 
-        with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe:
-            return pandas.read_csv(
-                pipe,
-                index_col=index_col,
-                parse_dates=parse_dates,
-                dtype=dtype,
-                nrows=nrows,
+            writer = functools.partial(
+                cursor.copy_expert,
+                'COPY {source} TO STDOUT WITH CSV HEADER'.format(
+                    source=source),
             )
 
+            with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe:
+                return pandas.read_csv(
+                    pipe,
+                    index_col=index_col,
+                    parse_dates=parse_dates,
+                    dtype=dtype,
+                    nrows=nrows,
+                )
+
 
 @pandas.api.extensions.register_dataframe_accessor('pg_copy_from')
 @functools.wraps(data_frame_pg_copy_from)

test/ext_test/pandas_test.py

@@ -8,13 +8,6 @@
 import ohio.ext.pandas  # noqa
 
 
-def get_connectable(engine, use_conn):
-    return engine.connect() if use_conn else engine
-
-
-parametrize_connectable = pytest.mark.parametrize('use_conn', (True, False))
-
-
 class TestPandasExtPgCopyTo:
 
     names = ('Alice', 'Bob', 'Conner', 'Denise')
@@ -44,11 +37,10 @@ def test_dataframe_object_dir(self, df):
         assert 'pg_copy_to' in members
         assert 'pg_copy_from' not in members
 
-    @parametrize_connectable
-    def test_pg_copy_to(self, engine, df, use_conn):
+    def test_pg_copy_to(self, engine, df):
         assert not self.table_exists('users', engine)
 
-        df.pg_copy_to('users', get_connectable(engine, use_conn))
+        df.pg_copy_to('users', engine)
 
         assert self.table_exists('users', engine)
 
@@ -59,9 +51,8 @@ def test_pg_copy_to(self, engine, df, use_conn):
 
 class TestPandasExtPgCopyFrom:
 
-    @parametrize_connectable
     @pytest.mark.parametrize('schema', (None, 'test_copy_from'))
-    def test_pg_copy_from(self, engine, schema, use_conn):
+    def test_pg_copy_from(self, engine, schema):
         users = (
             ('Alice', datetime(2019, 1, 2, 13, 0, 0), 302.1),
             ('Bob', datetime(2018, 10, 20, 8, 7, 10), 2.4),
@@ -96,7 +87,7 @@ def test_pg_copy_from(self, engine, schema, use_conn):
 
         df = pandas.DataFrame.pg_copy_from(
             'users',
-            get_connectable(engine, use_conn),
+            engine,
             schema=schema,
             index_col='id',
             parse_dates=['last_login'],
@@ -106,7 +97,8 @@ def test_pg_copy_from(self, engine, schema, use_conn):
         assert df.index.dtype == 'int64'
         assert df.index.tolist() == [1, 2, 3, 4]
 
-        assert df.columns.tolist() == ['name', 'last_login', 'tetris_high_score']
+        assert df.columns.tolist() == [
+            'name', 'last_login', 'tetris_high_score']
         assert df.dtypes.to_dict() == {
             'name': 'object',
             'last_login': 'datetime64[ns]',