From c50b600810c7e2d8ca776a6974f2d973005e19b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20De=20Un=C3=A1nue?= Date: Fri, 10 Jun 2022 01:40:49 -0400 Subject: [PATCH 1/3] guix manifest --- channels-lock.scm | 45 +++++++++++++++++++++++++++++++++++++++ manifest.scm | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 channels-lock.scm create mode 100644 manifest.scm diff --git a/channels-lock.scm b/channels-lock.scm new file mode 100644 index 0000000..47b9d91 --- /dev/null +++ b/channels-lock.scm @@ -0,0 +1,45 @@ +(list (channel + (name 'guix) + (url "https://git.savannah.gnu.org/git/guix.git") + (branch "master") + (commit + "4ba4cb1d49d70a00f7236f60bd92e5eccef573dd") + (introduction + (make-channel-introduction + "9edb3f66fd807b096b48283debdcddccfea34bad" + (openpgp-fingerprint + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name 'rde) + (url "https://git.sr.ht/~abcdw/rde") + (branch "master") + (commit + "24f632d8c68ae900732c8a54a6344a10f37de6d9") + (introduction + (make-channel-introduction + "257cebd587b66e4d865b3537a9a88cccd7107c95" + (openpgp-fingerprint + "2841 9AC6 5038 7440 C7E9 2FFA 2208 D209 58C1 DEB0")))) + (channel + (name 'nano-channel) + (url "file:///home/nanounanue/projects/nano-channel") + (branch "main") + (commit + "256b5c0291346d1f10dbe6d340fa476d1a95e869")) + (channel + (name 'nonguix) + (url "https://gitlab.com/nonguix/nonguix") + (branch "master") + (commit + "e87ff5f3a24b8a0696b8259594ac4fcf6fce0bc3")) + (channel + (name 'flat) + (url "https://github.com/flatwhatson/guix-channel.git") + (branch "master") + (commit + "657da22f0229b978b7bf4e4d476f59f17f6a175f") + (introduction + (make-channel-introduction + "33f86a4b48205c0dc19d7c036c85393f0766f806" + (openpgp-fingerprint + "736A C00E 1254 378B A982 7AF6 9DBE 8265 81B6 4490"))))) diff --git a/manifest.scm b/manifest.scm new file mode 100644 index 0000000..01e6a4b --- /dev/null +++ b/manifest.scm @@ -0,0 +1,54 @@ +(specifications->manifest + '(;; Python + "python" + "httpie" + ;; "python-pip" + "jupyter" + + + ;; pylsp + "python-rope" + "python-flake8" + "python-mccabe" + "python-pycodestyle" + "python-pydocstyle" + "python-pylint" + "python-isort" + "python-black" + "python-lsp-server" + "python-yapf" + "python-autopep8" + "python-xarray" + + ;; ext + "python-pandas" + "python-numpy" + + ;; docs + "python-sphinx" + + ;; profile + "python-matplotlib" + + ;; test + "python-sqlalchemy" + "python-ipython-sql" + "python-psycopg2-binary" + "python-pytest" + "python-nose" + + ;; dev + "python-bump2version" + "python-tox" + "python-twine" + "python-wheel" + + ;; GNU utils + "make" + "gcc-toolchain" + ;; CLI + "tmux" + "tmate" + ;; postgres + "postgresql" + )) From a9e3d9552826acbe61b832fbd93a564f9a41ce59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20De=20Un=C3=A1nue?= Date: Fri, 10 Jun 2022 01:41:16 -0400 Subject: [PATCH 2/3] Updated to SQLAlchemy 1.4.x - Added context manager, so no need to do it when use it - Removed connection as argument. Now only supports Engine. - Updated requirements (Following the versions from the channels-lock.scm) --- requirement/dev.txt | 8 +++---- requirement/docs.txt | 2 +- requirement/include/ext.txt | 4 ++-- requirement/profile.txt | 4 ++-- requirement/test.txt | 6 ++--- src/ohio/ext/pandas.py | 46 ++++++++++++++++++++---------------- test/ext_test/pandas_test.py | 20 +++++----------- 7 files changed, 43 insertions(+), 47 deletions(-) diff --git a/requirement/dev.txt b/requirement/dev.txt index f700453..24169b6 100644 --- a/requirement/dev.txt +++ b/requirement/dev.txt @@ -1,5 +1,5 @@ argcmdr==0.6.0 -bump2version==0.5.10 -tox==3.7.0 -twine==1.13.0 -wheel==0.33.1 +bump2version==1.0.1 +tox==3.20.0 +twine==1.15.0 +wheel==0.37.0 diff --git a/requirement/docs.txt b/requirement/docs.txt index 09df422..066bc3b 100644 --- a/requirement/docs.txt +++ b/requirement/docs.txt @@ -1,3 +1,3 @@ -r ./include/ext.txt https://github.com/dssg/restbuilder/archive/jsl.zip -sphinx==2.0.0 +sphinx==4.5.0 diff --git a/requirement/include/ext.txt b/requirement/include/ext.txt index 800fa49..df38c54 100644 --- a/requirement/include/ext.txt +++ b/requirement/include/ext.txt @@ -1,2 +1,2 @@ -numpy==1.16.2 -pandas==0.24.1 +numpy==1.21.6 +pandas==1.4.2 diff --git a/requirement/profile.txt b/requirement/profile.txt index a4672aa..72ac2de 100644 --- a/requirement/profile.txt +++ b/requirement/profile.txt @@ -1,5 +1,5 @@ -r ./include/ext.txt -matplotlib==3.0.3 +matplotlib==3.5.2 memory-profiler==0.55.0 -sqlalchemy==1.3.1 +sqlalchemy==1.4.35 testing.postgresql==1.3.0 diff --git a/requirement/test.txt b/requirement/test.txt index 2486cb0..7b18745 100644 --- a/requirement/test.txt +++ b/requirement/test.txt @@ -1,6 +1,6 @@ -r ./include/ext.txt -psycopg2-binary==2.7.7 -pytest==4.3.0 +psycopg2-binary==2.9.3 +pytest==6.2.5 signalled-timeout==1.0.0 -sqlalchemy==1.3.1 +sqlalchemy==1.4.35 testing.postgresql==1.3.0 diff --git a/src/ohio/ext/pandas.py b/src/ohio/ext/pandas.py index 5f4b48a..a8cc8eb 100644 --- a/src/ohio/ext/pandas.py +++ b/src/ohio/ext/pandas.py @@ -60,6 +60,7 @@ class DataFramePgCopyTo: (excepting parameter ``method``). """ + def __init__(self, data_frame): self.data_frame = data_frame @@ -72,7 +73,7 @@ def __call__(self, *args, **kwargs): ) -def to_sql_method_pg_copy_to(table, conn, keys, data_iter): +def to_sql_method_pg_copy_to(table, engine, keys, data_iter): """Write pandas data to table via stream through PostgreSQL ``COPY``. @@ -91,12 +92,13 @@ def to_sql_method_pg_copy_to(table, conn, keys, data_iter): columns=columns, ) - with ohio.CsvTextIO(data_iter) as csv_buffer, \ - conn.connection.cursor() as cursor: - cursor.copy_expert(sql, csv_buffer) + with ohio.CsvTextIO(data_iter) as csv_buffer: + with engine.connect() as conn: + with conn.connection.cursor() as cursor: + cursor.copy_expert(sql, csv_buffer) -def data_frame_pg_copy_from(sql, connectable, +def data_frame_pg_copy_from(sql, engine, schema=None, index_col=None, parse_dates=False, columns=None, dtype=None, nrows=None, buffer_size=100): @@ -131,7 +133,7 @@ def data_frame_pg_copy_from(sql, connectable, ``100``. """ - pandas_sql = pandas.io.sql.SQLDatabase(connectable) + pandas_sql = pandas.io.sql.SQLDatabase(engine) try: is_table_name = pandas_sql.has_table(sql, schema) @@ -158,30 +160,32 @@ def data_frame_pg_copy_from(sql, connectable, else: schema_description = 'default schema' - hint = " (hint: no such table {!r} found under {})".format(sql, schema_description) + hint = " (hint: no such table {!r} found under {})".format( + sql, schema_description) raise TypeError("keyword arguments 'columns' and 'schema' supported " "only when copying from table" + hint) else: source = "({})".format(sql) - with connectable.connect() as conn: - cursor = conn.connection.cursor() - - writer = functools.partial( - cursor.copy_expert, - 'COPY {source} TO STDOUT WITH CSV HEADER'.format(source=source), - ) + with engine.connect() as conn: + with conn.connection.cursor() as cursor: - with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe: - return pandas.read_csv( - pipe, - index_col=index_col, - parse_dates=parse_dates, - dtype=dtype, - nrows=nrows, + writer = functools.partial( + cursor.copy_expert, + 'COPY {source} TO STDOUT WITH CSV HEADER'.format( + source=source), ) + with ohio.pipe_text(writer, buffer_size=buffer_size) as pipe: + return pandas.read_csv( + pipe, + index_col=index_col, + parse_dates=parse_dates, + dtype=dtype, + nrows=nrows, + ) + @pandas.api.extensions.register_dataframe_accessor('pg_copy_from') @functools.wraps(data_frame_pg_copy_from) diff --git a/test/ext_test/pandas_test.py b/test/ext_test/pandas_test.py index eb621c8..e69fa3b 100644 --- a/test/ext_test/pandas_test.py +++ b/test/ext_test/pandas_test.py @@ -8,13 +8,6 @@ import ohio.ext.pandas # noqa -def get_connectable(engine, use_conn): - return engine.connect() if use_conn else engine - - -parametrize_connectable = pytest.mark.parametrize('use_conn', (True, False)) - - class TestPandasExtPgCopyTo: names = ('Alice', 'Bob', 'Conner', 'Denise') @@ -44,11 +37,10 @@ def test_dataframe_object_dir(self, df): assert 'pg_copy_to' in members assert 'pg_copy_from' not in members - @parametrize_connectable - def test_pg_copy_to(self, engine, df, use_conn): + def test_pg_copy_to(self, engine, df): assert not self.table_exists('users', engine) - df.pg_copy_to('users', get_connectable(engine, use_conn)) + df.pg_copy_to('users', engine) assert self.table_exists('users', engine) @@ -59,9 +51,8 @@ def test_pg_copy_to(self, engine, df, use_conn): class TestPandasExtPgCopyFrom: - @parametrize_connectable @pytest.mark.parametrize('schema', (None, 'test_copy_from')) - def test_pg_copy_from(self, engine, schema, use_conn): + def test_pg_copy_from(self, engine, schema): users = ( ('Alice', datetime(2019, 1, 2, 13, 0, 0), 302.1), ('Bob', datetime(2018, 10, 20, 8, 7, 10), 2.4), @@ -96,7 +87,7 @@ def test_pg_copy_from(self, engine, schema, use_conn): df = pandas.DataFrame.pg_copy_from( 'users', - get_connectable(engine, use_conn), + engine, schema=schema, index_col='id', parse_dates=['last_login'], @@ -106,7 +97,8 @@ def test_pg_copy_from(self, engine, schema, use_conn): assert df.index.dtype == 'int64' assert df.index.tolist() == [1, 2, 3, 4] - assert df.columns.tolist() == ['name', 'last_login', 'tetris_high_score'] + assert df.columns.tolist() == [ + 'name', 'last_login', 'tetris_high_score'] assert df.dtypes.to_dict() == { 'name': 'object', 'last_login': 'datetime64[ns]', From b935c50d776c28335ec0b220e1f7808d02f25623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20De=20Un=C3=A1nue?= Date: Thu, 16 Jun 2022 00:38:14 -0500 Subject: [PATCH 3/3] =?UTF-8?q?Bump=20version:=200.5.0=20=E2=86=92=200.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.rst | 4 ++-- channels-lock.scm | 45 ------------------------------------ doc/conf.py | 2 +- doc/index.rst | 4 ++-- manifest.scm | 54 -------------------------------------------- setup.cfg | 3 +-- setup.py | 2 +- src/ohio/__init__.py | 2 +- 8 files changed, 8 insertions(+), 108 deletions(-) delete mode 100644 channels-lock.scm delete mode 100644 manifest.scm diff --git a/README.rst b/README.rst index 8035376..9e36d02 100644 --- a/README.rst +++ b/README.rst @@ -803,7 +803,7 @@ of 896,677 rows across 83 columns: 1 of these of type timestamp, 51 integers and 31 floats. The benchmarking package, ``prof``, is preserved in `Ohio's repository `_. -.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true +.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true ohio_pg_copy_from_X ``pg_copy_from(buffer_size=X)`` @@ -831,7 +831,7 @@ pandas_read_csv_stringio ``COPY`` to a ``StringIO``, from which pandas constructs a ``DataFrame``. -.. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true +.. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true ohio_pg_copy_to ``pg_copy_to()`` diff --git a/channels-lock.scm b/channels-lock.scm deleted file mode 100644 index 47b9d91..0000000 --- a/channels-lock.scm +++ /dev/null @@ -1,45 +0,0 @@ -(list (channel - (name 'guix) - (url "https://git.savannah.gnu.org/git/guix.git") - (branch "master") - (commit - "4ba4cb1d49d70a00f7236f60bd92e5eccef573dd") - (introduction - (make-channel-introduction - "9edb3f66fd807b096b48283debdcddccfea34bad" - (openpgp-fingerprint - "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) - (channel - (name 'rde) - (url "https://git.sr.ht/~abcdw/rde") - (branch "master") - (commit - "24f632d8c68ae900732c8a54a6344a10f37de6d9") - (introduction - (make-channel-introduction - "257cebd587b66e4d865b3537a9a88cccd7107c95" - (openpgp-fingerprint - "2841 9AC6 5038 7440 C7E9 2FFA 2208 D209 58C1 DEB0")))) - (channel - (name 'nano-channel) - (url "file:///home/nanounanue/projects/nano-channel") - (branch "main") - (commit - "256b5c0291346d1f10dbe6d340fa476d1a95e869")) - (channel - (name 'nonguix) - (url "https://gitlab.com/nonguix/nonguix") - (branch "master") - (commit - "e87ff5f3a24b8a0696b8259594ac4fcf6fce0bc3")) - (channel - (name 'flat) - (url "https://github.com/flatwhatson/guix-channel.git") - (branch "master") - (commit - "657da22f0229b978b7bf4e4d476f59f17f6a175f") - (introduction - (make-channel-introduction - "33f86a4b48205c0dc19d7c036c85393f0766f806" - (openpgp-fingerprint - "736A C00E 1254 378B A982 7AF6 9DBE 8265 81B6 4490"))))) diff --git a/doc/conf.py b/doc/conf.py index 32ddbf8..04a9fa6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -23,7 +23,7 @@ author = 'Jesse London' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.6.0' # -- General configuration --------------------------------------------------- diff --git a/doc/index.rst b/doc/index.rst index 1bde129..2f507be 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -113,7 +113,7 @@ Modules .. Also note the query string ?sanitize=true This is necessary to ensure that SVG is returned with the image content type. - .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true + .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-database-to-datafram-1554345457.svg?sanitize=true ohio_pg_copy_from_X ``pg_copy_from(buffer_size=X)`` @@ -141,7 +141,7 @@ Modules ``COPY`` to a ``StringIO``, from which pandas constructs a ``DataFrame``. - .. image:: https://raw.githubusercontent.com/dssg/ohio/0.5.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true + .. image:: https://raw.githubusercontent.com/dssg/ohio/0.6.0/doc/img/profile-copy-from-dataframe-to-databas-1555458507.svg?sanitize=true ohio_pg_copy_to ``pg_copy_to()`` diff --git a/manifest.scm b/manifest.scm deleted file mode 100644 index 01e6a4b..0000000 --- a/manifest.scm +++ /dev/null @@ -1,54 +0,0 @@ -(specifications->manifest - '(;; Python - "python" - "httpie" - ;; "python-pip" - "jupyter" - - - ;; pylsp - "python-rope" - "python-flake8" - "python-mccabe" - "python-pycodestyle" - "python-pydocstyle" - "python-pylint" - "python-isort" - "python-black" - "python-lsp-server" - "python-yapf" - "python-autopep8" - "python-xarray" - - ;; ext - "python-pandas" - "python-numpy" - - ;; docs - "python-sphinx" - - ;; profile - "python-matplotlib" - - ;; test - "python-sqlalchemy" - "python-ipython-sql" - "python-psycopg2-binary" - "python-pytest" - "python-nose" - - ;; dev - "python-bump2version" - "python-tox" - "python-twine" - "python-wheel" - - ;; GNU utils - "make" - "gcc-toolchain" - ;; CLI - "tmux" - "tmate" - ;; postgres - "postgresql" - )) diff --git a/setup.cfg b/setup.cfg index 6f59a81..7d97d7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ commit = True tag = True tag_name = {new_version} -current_version = 0.5.0 +current_version = 0.6.0 [bumpversion:file:setup.py] @@ -19,4 +19,3 @@ test = pytest [metadata] license_files = LICENSE - diff --git a/setup.py b/setup.py index ac8a196..380122b 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def stream_requirements(fd): setup( name='ohio', - version='0.5.0', + version='0.6.0', description="I/O extras", long_description=README_PATH.read_text(), long_description_content_type='text/x-rst', diff --git a/src/ohio/__init__.py b/src/ohio/__init__.py index b6294fa..108beb1 100644 --- a/src/ohio/__init__.py +++ b/src/ohio/__init__.py @@ -35,4 +35,4 @@ ) -__version__ = '0.5.0' +__version__ = '0.6.0'