diff --git a/README.rst b/README.rst index 8035376..9e36d02 100644 --- a/README.rst +++ b/README.rst @@ -803,7 +803,7 @@ of 896,677 rows across 83 columns: 1 of these of type timestamp, 51 integers and 31 floats. The benchmarking package, ``prof``, is preserved in `Ohio's repository `_. -.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true +.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true ohio_pg_copy_from_X ``pg_copy_from(buffer_size=X)`` @@ -831,7 +831,7 @@ pandas_read_csv_stringio ``COPY`` to a ``StringIO``, from which pandas constructs a ``DataFrame``. -.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true +.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true ohio_pg_copy_to ``pg_copy_to()`` diff --git a/doc/conf.py b/doc/conf.py index 32ddbf8..04a9fa6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -23,7 +23,7 @@ author = 'Jesse London' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.6.0' # -- General configuration --------------------------------------------------- diff --git a/doc/index.rst b/doc/index.rst index 1bde129..2f507be 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -113,7 +113,7 @@ Modules .. Also note the query string ?sanitize=true This is necessary to ensure that SVG is returned with the image content type. - .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true + .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true ohio_pg_copy_from_X ``pg_copy_from(buffer_size=X)`` @@ -141,7 +141,7 @@ Modules ``COPY`` to a ``StringIO``, from which pandas constructs a ``DataFrame``. - .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true + .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true ohio_pg_copy_to ``pg_copy_to()`` diff --git a/requirement/dev.txt b/requirement/dev.txt index f700453..24169b6 100644 --- a/requirement/dev.txt +++ b/requirement/dev.txt @@ -1,5 +1,5 @@ argcmdr==0.6.0 -bump2version==0.5.10 -tox==3.7.0 -twine==1.13.0 -wheel==0.33.1 +bump2version==1.0.1 +tox==3.20.0 +twine==1.15.0 +wheel==0.37.0 diff --git a/requirement/docs.txt b/requirement/docs.txt index 09df422..066bc3b 100644 --- a/requirement/docs.txt +++ b/requirement/docs.txt @@ -1,3 +1,3 @@ -r ./include/ext.txt https://github.com/dssg/restbuilder/archive/jsl.zip -sphinx==2.0.0 +sphinx==4.5.0 diff --git a/requirement/include/ext.txt b/requirement/include/ext.txt index 800fa49..df38c54 100644 --- a/requirement/include/ext.txt +++ b/requirement/include/ext.txt @@ -1,2 +1,2 @@ -numpy==1.16.2 -pandas==0.24.1 +numpy==1.21.6 +pandas==1.4.2 diff --git a/requirement/profile.txt b/requirement/profile.txt index a4672aa..72ac2de 100644 --- a/requirement/profile.txt +++ b/requirement/profile.txt @@ -1,5 +1,5 @@ -r ./include/ext.txt -matplotlib==3.0.3 +matplotlib==3.5.2 memory-profiler==0.55.0 -sqlalchemy==1.3.1 +sqlalchemy==1.4.35 testing.postgresql==1.3.0 diff --git a/requirement/test.txt b/requirement/test.txt index 2486cb0..7b18745 100644 --- a/requirement/test.txt +++ b/requirement/test.txt @@ -1,6 +1,6 @@ -r ./include/ext.txt -psycopg2-binary==2.7.7 -pytest==4.3.0 +psycopg2-binary==2.9.3 +pytest==6.2.5 signalled-timeout==1.0.0 -sqlalchemy==1.3.1 +sqlalchemy==1.4.35 testing.postgresql==1.3.0 diff --git a/setup.cfg b/setup.cfg index 6f59a81..7d97d7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ commit = True tag = True tag_name = {new_version} -current_version = 0.5.0 +current_version = 0.6.0 [bumpversion:file:setup.py] @@ -19,4 +19,3 @@ test = pytest [metadata] license_files = LICENSE - diff --git a/setup.py b/setup.py index ac8a196..380122b 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def stream_requirements(fd): setup( name='ohio', - version='0.5.0', + version='0.6.0', description="I/O extras", long_description=README_PATH.read_text(), long_description_content_type='text/x-rst', diff --git a/src/ohio/__init__.py b/src/ohio/__init__.py index b6294fa..108beb1 100644 --- a/src/ohio/__init__.py +++ b/src/ohio/__init__.py @@ -35,4 +35,4 @@ ) -__version__ = '0.5.0' +__version__ = '0.6.0' diff --git a/src/ohio/ext/pandas.py b/src/ohio/ext/pandas.py index 5f4b48a..a8cc8eb 100644 --- a/src/ohio/ext/pandas.py +++ b/src/ohio/ext/pandas.py @@ -60,6 +60,7 @@ class DataFramePgCopyTo: (excepting parameter ``method``). """ + def __init__(self, data_frame): self.data_frame = data_frame @@ -72,7 +73,7 @@ def __call__(self, *args, **kwargs): ) -def to_sql_method_pg_copy_to(table, conn, keys, data_iter): +def to_sql_method_pg_copy_to(table, engine, keys, data_iter): """Write pandas data to table via stream through PostgreSQL ``COPY``. @@ -91,12 +92,13 @@ def to_sql_method_pg_copy_to(table, conn, keys, data_iter): columns=columns, ) - with ohio.CsvTextIO(data_iter) as csv_buffer, \ - conn.connection.cursor() as cursor: - cursor.copy_expert(sql, csv_buffer) + with ohio.CsvTextIO(data_iter) as csv_buffer: + with engine.connect() as conn: + with conn.connection.cursor() as cursor: + cursor.copy_expert(sql, csv_buffer) -def data_frame_pg_copy_from(sql, connectable, +def data_frame_pg_copy_from(sql, engine, schema=None, index_col=None, parse_dates=False, columns=None, dtype=None, nrows=None, buffer_size=100): @@ -131,7 +133,7 @@ def data_frame_pg_copy_from(sql, connectable, ``100``. """ - pandas_sql = pandas.io.sql.SQLDatabase(connectable) + pandas_sql = pandas.io.sql.SQLDatabase(engine) try: is_table_name = pandas_sql.has_table(sql, schema) @@ -158,30 +160,32 @@ def data_frame_pg_copy_from(sql, connectable, else: schema_description = 'default schema' - hint = " (hint: no such table {!r} found under {})".format(sql, schema_description) + hint = " (hint: no such table {!r} found under {})".format( + sql, schema_description) raise TypeError("keyword arguments 'columns' and 'schema' supported " "only when copying from table" + hint) else: source = "({})".format(sql) - with connectable.connect() as conn: - cursor = conn.connection.cursor() - - writer = functools.partial( - cursor.copy_expert, - 'COPY {source} TO STDOUT WITH CSV HEADER'.format(source=source), - ) + with engine.connect() as conn: + with conn.connection.cursor() as cursor: - with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe: - return pandas.read_csv( - pipe, - index_col=index_col, - parse_dates=parse_dates, - dtype=dtype, - nrows=nrows, + writer = functools.partial( + cursor.copy_expert, + 'COPY {source} TO STDOUT WITH CSV HEADER'.format( + source=source), ) + with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe: + return pandas.read_csv( + pipe, + index_col=index_col, + parse_dates=parse_dates, + dtype=dtype, + nrows=nrows, + ) + @pandas.api.extensions.register_dataframe_accessor('pg_copy_from') @functools.wraps(data_frame_pg_copy_from) diff --git a/test/ext_test/pandas_test.py b/test/ext_test/pandas_test.py index eb621c8..e69fa3b 100644 --- a/test/ext_test/pandas_test.py +++ b/test/ext_test/pandas_test.py @@ -8,13 +8,6 @@ import ohio.ext.pandas # noqa -def get_connectable(engine, use_conn): - return engine.connect() if use_conn else engine - - -parametrize_connectable = pytest.mark.parametrize('use_conn', (True, False)) - - class TestPandasExtPgCopyTo: names = ('Alice', 'Bob', 'Conner', 'Denise') @@ -44,11 +37,10 @@ def test_dataframe_object_dir(self, df): assert 'pg_copy_to' in members assert 'pg_copy_from' not in members - @parametrize_connectable - def test_pg_copy_to(self, engine, df, use_conn): + def test_pg_copy_to(self, engine, df): assert not self.table_exists('users', engine) - df.pg_copy_to('users', get_connectable(engine, use_conn)) + df.pg_copy_to('users', engine) assert self.table_exists('users', engine) @@ -59,9 +51,8 @@ def test_pg_copy_to(self, engine, df, use_conn): class TestPandasExtPgCopyFrom: - @parametrize_connectable @pytest.mark.parametrize('schema', (None, 'test_copy_from')) - def test_pg_copy_from(self, engine, schema, use_conn): + def test_pg_copy_from(self, engine, schema): users = ( ('Alice', datetime(2019, 1, 2, 13, 0, 0), 302.1), ('Bob', datetime(2018, 10, 20, 8, 7, 10), 2.4), @@ -96,7 +87,7 @@ def test_pg_copy_from(self, engine, schema, use_conn): df = pandas.DataFrame.pg_copy_from( 'users', - get_connectable(engine, use_conn), + engine, schema=schema, index_col='id', parse_dates=['last_login'], @@ -106,7 +97,8 @@ def test_pg_copy_from(self, engine, schema, use_conn): assert df.index.dtype == 'int64' assert df.index.tolist() == [1, 2, 3, 4] - assert df.columns.tolist() == ['name', 'last_login', 'tetris_high_score'] + assert df.columns.tolist() == [ + 'name', 'last_login', 'tetris_high_score'] assert df.dtypes.to_dict() == { 'name': 'object', 'last_login': 'datetime64[ns]',