From 8bd13099f9398efa503edd14f3816b73a97fcc81 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 21 Dec 2023 01:02:01 +0100 Subject: [PATCH] Split dialect from driver, part 3: Re-activate integration-/doctests --- .github/workflows/nightly.yml | 41 ++-- .github/workflows/tests.yml | 32 ++- DEVELOP.md | 13 +- bootstrap.sh | 2 +- .../sqlalchemy => }/advanced-querying.rst | 8 +- docs/by-example/index.rst | 30 --- .../by-example/sqlalchemy/getting-started.rst | 211 ------------------ docs/{by-example/sqlalchemy => }/crud.rst | 8 +- .../{by-example/sqlalchemy => }/dataframe.rst | 10 +- docs/getting-started.rst | 211 +++++++++++++++--- docs/index-all.rst | 18 -- docs/index.rst | 32 ++- .../sqlalchemy => }/inspection-reflection.rst | 8 +- docs/install.rst | 52 +++++ docs/{sqlalchemy.rst => overview.rst} | 8 +- .../sqlalchemy => }/working-with-types.rst | 8 +- pyproject.toml | 1 + .../testing/testdata/mappings/locations.sql | 13 -- .../testing/testdata/settings/test_a.json | 6 - .../assets/locations.jsonl | 0 tests/assets/locations.sql | 13 ++ tests/docker-compose.yml | 14 ++ .../client/tests.py => tests/integration.py | 127 ++++++----- 23 files changed, 432 insertions(+), 434 deletions(-) rename docs/{by-example/sqlalchemy => }/advanced-querying.rst (98%) delete mode 100644 docs/by-example/index.rst delete mode 100644 docs/by-example/sqlalchemy/getting-started.rst rename docs/{by-example/sqlalchemy => }/crud.rst (97%) rename docs/{by-example/sqlalchemy => }/dataframe.rst (98%) delete mode 100644 docs/index-all.rst rename docs/{by-example/sqlalchemy => }/inspection-reflection.rst (94%) create mode 100644 docs/install.rst rename docs/{sqlalchemy.rst => overview.rst} (99%) rename docs/{by-example/sqlalchemy => }/working-with-types.rst (97%) delete mode 100644 src/crate/testing/testdata/mappings/locations.sql delete mode 100644 src/crate/testing/testdata/settings/test_a.json rename src/crate/testing/testdata/data/test_a.json => tests/assets/locations.jsonl (100%) create mode 100644 tests/assets/locations.sql create mode 100644 tests/docker-compose.yml rename src/crate/client/tests.py => tests/integration.py (63%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b35c7d23..f585f3ee 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,10 +8,11 @@ on: jobs: nightly: - name: "Python: ${{ matrix.python-version }} - SQLA: ${{ matrix.sqla-version }} - CrateDB: ${{ matrix.cratedb-version }} - on ${{ matrix.os }}" + name: " + Python: ${{ matrix.python-version }} + SQLAlchemy: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + " runs-on: ${{ matrix.os }} strategy: matrix: @@ -37,19 +38,34 @@ jobs: PIP_ALLOW_PRERELEASE: ${{ matrix.pip-allow-prerelease }} steps: - - uses: actions/checkout@v4 + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Run CrateDB + run: docker compose -f tests/docker-compose.yml up -d + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + architecture: x64 cache: 'pip' - cache-dependency-path: 'setup.py' + cache-dependency-path: + pyproject.toml - - name: Invoke tests + - name: Set up project run: | - # Propagate build matrix information. - ./devtools/setup_ci.sh + # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). + # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400 + pip install "setuptools>=64" --upgrade + + # Install package in editable mode. + pip install --use-pep517 --prefer-binary --editable='.[develop,test]' + + - name: Invoke tests + run: | # Bootstrap environment. source bootstrap.sh @@ -57,9 +73,6 @@ jobs: # Report about the test matrix slot. echo "Invoking tests with CrateDB ${CRATEDB_VERSION} and SQLAlchemy ${SQLALCHEMY_VERSION}" - # Run linter. - flake8 src bin - - # Run tests. + # Run linters and software tests. export SQLALCHEMY_WARN_20=1 - bin/test -vvv + poe check diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ad03d81..70721b69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,41 +12,30 @@ concurrency: jobs: test: - name: "Python: ${{ matrix.python-version }} - SQLA: ${{ matrix.sqla-version }} - on ${{ matrix.os }}" + name: " + Python: ${{ matrix.python-version }} + SQLAlchemy: ${{ matrix.sqla-version }} + " runs-on: ${{ matrix.os }} strategy: matrix: - os: ['ubuntu-latest', 'macos-latest'] + os: ['ubuntu-latest'] python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - cratedb-version: ['5.4.5'] + cratedb-version: ['5.5.1'] sqla-version: ['<1.4', '<1.5', '<2.1'] pip-allow-prerelease: ['false'] exclude: - # To save resources, only use the most recent Python versions on macOS. - - os: 'macos-latest' - python-version: '3.7' - - os: 'macos-latest' - python-version: '3.8' - - os: 'macos-latest' - python-version: '3.9' - - os: 'macos-latest' - python-version: '3.10' # SQLAlchemy 1.3 is not supported on Python 3.12 and higher. - os: 'ubuntu-latest' python-version: '3.12' sqla-version: '<1.4' - - os: 'macos-latest' - python-version: '3.12' - sqla-version: '<1.4' # Another CI test matrix slot to test against prerelease versions of Python packages. include: - os: 'ubuntu-latest' python-version: '3.12' - cratedb-version: '5.4.5' + cratedb-version: '5.5.1' sqla-version: 'latest' pip-allow-prerelease: 'true' @@ -59,7 +48,12 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} steps: - - uses: actions/checkout@v4 + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Run CrateDB + run: docker compose -f tests/docker-compose.yml up -d - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/DEVELOP.md b/DEVELOP.md index bfac485e..bf5a29d0 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -12,15 +12,11 @@ This command should automatically install all prerequisites for the development sandbox and drop you into the virtualenv, ready for invoking further commands. -## Running tests +## Running Tests -All tests will be invoked using the Python interpreter that was used -when creating the Python virtualenv. - -Some examples how to invoke the test runner are outlined below. - -Run all tests: +Verify code by running all linters and software tests: + docker compose -f tests/docker-compose.yml up poe check Run specific tests: @@ -28,6 +24,9 @@ Run specific tests: pytest -k SqlAlchemyCompilerTest pytest -k test_score + # Integration tests, written as doctests. + python -m unittest -vvv tests/integration.py + Format code: poe format diff --git a/bootstrap.sh b/bootstrap.sh index 06212f7b..380f70cf 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -17,7 +17,7 @@ # set -x # Default variables. -CRATEDB_VERSION=${CRATEDB_VERSION:-5.2.2} +CRATEDB_VERSION=${CRATEDB_VERSION:-5.5.1} SQLALCHEMY_VERSION=${SQLALCHEMY_VERSION:-<2.1} diff --git a/docs/by-example/sqlalchemy/advanced-querying.rst b/docs/advanced-querying.rst similarity index 98% rename from docs/by-example/sqlalchemy/advanced-querying.rst rename to docs/advanced-querying.rst index 7c4d6781..a3a00096 100644 --- a/docs/by-example/sqlalchemy/advanced-querying.rst +++ b/docs/advanced-querying.rst @@ -1,8 +1,8 @@ -.. _sqlalchemy-advanced-querying: +.. _advanced-querying: -============================= -SQLAlchemy: Advanced querying -============================= +================= +Advanced querying +================= This section of the documentation demonstrates running queries using a fulltext index with an analyzer, queries using counting and aggregations, and support for diff --git a/docs/by-example/index.rst b/docs/by-example/index.rst deleted file mode 100644 index 301cf699..00000000 --- a/docs/by-example/index.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _by-example: - -########## -By example -########## - -This part of the documentation enumerates different kinds of examples how to -use the CrateDB Python client. - -.. _sqlalchemy-by-example: - -SQLAlchemy by example -===================== - -The examples in this section are all about CrateDB's `SQLAlchemy`_ dialect, and -its corresponding API interfaces, see also :ref:`sqlalchemy-support`. - -.. toctree:: - :maxdepth: 1 - - sqlalchemy/getting-started - sqlalchemy/crud - sqlalchemy/working-with-types - sqlalchemy/advanced-querying - sqlalchemy/inspection-reflection - sqlalchemy/dataframe - - -.. _Python DB API: https://peps.python.org/pep-0249/ -.. _SQLAlchemy: https://www.sqlalchemy.org/ diff --git a/docs/by-example/sqlalchemy/getting-started.rst b/docs/by-example/sqlalchemy/getting-started.rst deleted file mode 100644 index 33e8f75d..00000000 --- a/docs/by-example/sqlalchemy/getting-started.rst +++ /dev/null @@ -1,211 +0,0 @@ -.. _sqlalchemy-getting-started: - -=========================== -SQLAlchemy: Getting started -=========================== - -This section of the documentation shows how to connect to CrateDB using its -SQLAlchemy dialect, and how to run basic DDL statements based on an SQLAlchemy -ORM schema definition. - -Subsequent sections of the documentation will cover: - -- :ref:`sqlalchemy-crud` -- :ref:`sqlalchemy-working-with-types` -- :ref:`sqlalchemy-advanced-querying` -- :ref:`sqlalchemy-inspection-reflection` - - -.. rubric:: Table of Contents - -.. contents:: - :local: - - -Introduction -============ - -Import the relevant symbols: - - >>> import sqlalchemy as sa - >>> from sqlalchemy.orm import sessionmaker - >>> try: - ... from sqlalchemy.orm import declarative_base - ... except ImportError: - ... from sqlalchemy.ext.declarative import declarative_base - -Establish a connection to the database, see also :ref:`sa:engines_toplevel` -and :ref:`connect`: - - >>> engine = sa.create_engine(f"crate://{crate_host}") - >>> connection = engine.connect() - -Create an SQLAlchemy :doc:`Session `: - - >>> session = sessionmaker(bind=engine)() - >>> Base = declarative_base() - - -Connect -======= - -In SQLAlchemy, a connection is established using the ``create_engine`` function. -This function takes a connection string, actually an `URL`_, that varies from -database to database. - -In order to connect to a CrateDB cluster, the following connection strings are -valid: - - >>> sa.create_engine('crate://') - Engine(crate://) - -This will connect to the default server ('127.0.0.1:4200'). In order to connect -to a different server the following syntax can be used: - - >>> sa.create_engine('crate://otherserver:4200') - Engine(crate://otherserver:4200) - -Multiple Hosts --------------- -Because CrateDB is a clustered database running on multiple servers, it is -recommended to connect to all of them. This enables the DB-API layer to -use round-robin to distribute the load and skip a server if it becomes -unavailable. In order to make the driver aware of multiple servers, use -the ``connect_args`` parameter like so: - - >>> sa.create_engine('crate://', connect_args={ - ... 'servers': ['host1:4200', 'host2:4200'] - ... }) - Engine(crate://) - -TLS Options ------------ -As defined in :ref:`https_connection`, the client validates SSL server -certificates by default. To configure this further, use e.g. the ``ca_cert`` -attribute within the ``connect_args``, like: - - >>> ssl_engine = sa.create_engine( - ... 'crate://', - ... connect_args={ - ... 'servers': ['https://host1:4200'], - ... 'ca_cert': '/path/to/cacert.pem', - ... }) - -In order to disable SSL verification, use ``verify_ssl_cert = False``, like: - - >>> ssl_engine = sa.create_engine( - ... 'crate://', - ... connect_args={ - ... 'servers': ['https://host1:4200'], - ... 'verify_ssl_cert': False, - ... }) - -Timeout Options ---------------- -In order to configure TCP timeout options, use the ``timeout`` parameter within -``connect_args``, - - >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'timeout': 42.42}) - >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] - 42.42 - -or use the ``timeout`` URL parameter within the database connection URL. - - >>> timeout_engine = sa.create_engine('crate://localhost/?timeout=42.42') - >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] - 42.42 - -Pool Size ---------- - -In order to configure the database connection pool size, use the ``pool_size`` -parameter within ``connect_args``, - - >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'pool_size': 20}) - >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] - 20 - -or use the ``pool_size`` URL parameter within the database connection URL. - - >>> timeout_engine = sa.create_engine('crate://localhost/?pool_size=20') - >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] - 20 - - -Basic DDL operations -==================== - -.. note:: - - CrateDB currently does not know about different "databases". Instead, - tables can be created in different *schemas*. Schemas are created - implicitly on table creation and cannot be created explicitly. If a schema - does not exist yet, it will be created. - - The default CrateDB schema is ``doc``, and if you do not specify a schema, - this is what will be used. - - See also :ref:`schema-selection` and :ref:`crate-reference:ddl-create-table-schemas`. - - -Create tables -------------- - -First the table definition as class, using SQLAlchemy's :ref:`sa:orm_declarative_mapping`: - - >>> class Department(Base): - ... __tablename__ = 'departments' - ... __table_args__ = { - ... 'crate_number_of_replicas': '0' - ... } - ... id = sa.Column(sa.String, primary_key=True) - ... name = sa.Column(sa.String) - ... code = sa.Column(sa.Integer) - -As seen below, the table doesn't exist yet: - - >>> engine.dialect.has_table(connection, table_name='departments') - False - -In order to create all missing tables, the ``create_all`` method can be used: - - >>> Base.metadata.create_all(bind=engine) - -With that, the table has been created: - - >>> engine.dialect.has_table(connection, table_name='departments') - True - -Let's also verify that by inquiring the ``information_schema.columns`` table: - - >>> stmt = ("select table_name, column_name, ordinal_position, data_type " - ... "from information_schema.columns " - ... "where table_name = 'departments' " - ... "order by column_name") - >>> pprint([str(r) for r in connection.execute(sa.text(stmt))]) - ["('departments', 'code', 3, 'integer')", - "('departments', 'id', 1, 'text')", - "('departments', 'name', 2, 'text')"] - - -Drop tables ------------ - -In order to delete all tables reference within the ORM schema, invoke -``Base.metadata.drop_all()``. To delete a single table, use -``drop(...)``, as shown below: - - >>> Base.metadata.tables['departments'].drop(engine) - - >>> engine.dialect.has_table(connection, table_name='departments') - False - - -.. hidden: Disconnect from database - - >>> session.close() - >>> connection.close() - >>> engine.dispose() - - -.. _URL: https://en.wikipedia.org/wiki/Uniform_Resource_Locator diff --git a/docs/by-example/sqlalchemy/crud.rst b/docs/crud.rst similarity index 97% rename from docs/by-example/sqlalchemy/crud.rst rename to docs/crud.rst index 5a62df40..1e2e7264 100644 --- a/docs/by-example/sqlalchemy/crud.rst +++ b/docs/crud.rst @@ -1,8 +1,8 @@ -.. _sqlalchemy-crud: +.. _crud: -================================================ -SQLAlchemy: Create, retrieve, update, and delete -================================================ +==================================== +Create, retrieve, update, and delete +==================================== This section of the documentation shows how to query, insert, update and delete records using CrateDB's SQLAlchemy integration, it includes common scenarios diff --git a/docs/by-example/sqlalchemy/dataframe.rst b/docs/dataframe.rst similarity index 98% rename from docs/by-example/sqlalchemy/dataframe.rst rename to docs/dataframe.rst index a2be1f88..d516781c 100644 --- a/docs/by-example/sqlalchemy/dataframe.rst +++ b/docs/dataframe.rst @@ -1,9 +1,9 @@ -.. _sqlalchemy-pandas: -.. _sqlalchemy-dataframe: +.. _use-pandas: +.. _dataframe: -================================ -SQLAlchemy: DataFrame operations -================================ +==================== +DataFrame operations +==================== .. rubric:: Table of Contents diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 1125bece..838a515e 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -4,49 +4,208 @@ Getting started =============== -Learn how to install and get started with the Python client library for -`CrateDB`_. +This section of the documentation shows how to connect to CrateDB using its +SQLAlchemy dialect, and how to run basic DDL statements based on an SQLAlchemy +ORM schema definition. -.. rubric:: Table of contents +Subsequent sections of the documentation will cover: + +- :ref:`sqlalchemy-crud` +- :ref:`sqlalchemy-working-with-types` +- :ref:`sqlalchemy-advanced-querying` +- :ref:`sqlalchemy-inspection-reflection` + + +.. rubric:: Table of Contents .. contents:: :local: -Install + +Introduction +============ + +Import the relevant symbols: + + >>> import sqlalchemy as sa + >>> from sqlalchemy.orm import sessionmaker + >>> try: + ... from sqlalchemy.orm import declarative_base + ... except ImportError: + ... from sqlalchemy.ext.declarative import declarative_base + +Establish a connection to the database, see also :ref:`sa:engines_toplevel` +and :ref:`connect`: + + >>> engine = sa.create_engine(f"crate://{crate_host}") + >>> connection = engine.connect() + +Create an SQLAlchemy :doc:`Session `: + + >>> session = sessionmaker(bind=engine)() + >>> Base = declarative_base() + + +Connect ======= -.. highlight:: sh +In SQLAlchemy, a connection is established using the ``create_engine`` function. +This function takes a connection string, actually an `URL`_, that varies from +database to database. + +In order to connect to a CrateDB cluster, the following connection strings are +valid: + + >>> sa.create_engine('crate://') + Engine(crate://) + +This will connect to the default server ('127.0.0.1:4200'). In order to connect +to a different server the following syntax can be used: + + >>> sa.create_engine('crate://otherserver:4200') + Engine(crate://otherserver:4200) + +Multiple Hosts +-------------- +Because CrateDB is a clustered database running on multiple servers, it is +recommended to connect to all of them. This enables the DB-API layer to +use round-robin to distribute the load and skip a server if it becomes +unavailable. In order to make the driver aware of multiple servers, use +the ``connect_args`` parameter like so: + + >>> sa.create_engine('crate://', connect_args={ + ... 'servers': ['host1:4200', 'host2:4200'] + ... }) + Engine(crate://) + +TLS Options +----------- +As defined in :ref:`https_connection`, the client validates SSL server +certificates by default. To configure this further, use e.g. the ``ca_cert`` +attribute within the ``connect_args``, like: + + >>> ssl_engine = sa.create_engine( + ... 'crate://', + ... connect_args={ + ... 'servers': ['https://host1:4200'], + ... 'ca_cert': '/path/to/cacert.pem', + ... }) + +In order to disable SSL verification, use ``verify_ssl_cert = False``, like: + + >>> ssl_engine = sa.create_engine( + ... 'crate://', + ... connect_args={ + ... 'servers': ['https://host1:4200'], + ... 'verify_ssl_cert': False, + ... }) + +Timeout Options +--------------- +In order to configure TCP timeout options, use the ``timeout`` parameter within +``connect_args``, + + >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'timeout': 42.42}) + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] + 42.42 + +or use the ``timeout`` URL parameter within the database connection URL. + + >>> timeout_engine = sa.create_engine('crate://localhost/?timeout=42.42') + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["timeout"] + 42.42 + +Pool Size +--------- + +In order to configure the database connection pool size, use the ``pool_size`` +parameter within ``connect_args``, + + >>> timeout_engine = sa.create_engine('crate://localhost/', connect_args={'pool_size': 20}) + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] + 20 + +or use the ``pool_size`` URL parameter within the database connection URL. + + >>> timeout_engine = sa.create_engine('crate://localhost/?pool_size=20') + >>> timeout_engine.raw_connection().driver_connection.client._pool_kw["maxsize"] + 20 + + +Basic DDL operations +==================== + +.. note:: + + CrateDB currently does not know about different "databases". Instead, + tables can be created in different *schemas*. Schemas are created + implicitly on table creation and cannot be created explicitly. If a schema + does not exist yet, it will be created. + + The default CrateDB schema is ``doc``, and if you do not specify a schema, + this is what will be used. + + See also :ref:`schema-selection` and :ref:`crate-reference:ddl-create-table-schemas`. + + +Create tables +------------- + +First the table definition as class, using SQLAlchemy's :ref:`sa:orm_declarative_mapping`: + + >>> class Department(Base): + ... __tablename__ = 'departments' + ... __table_args__ = { + ... 'crate_number_of_replicas': '0' + ... } + ... id = sa.Column(sa.String, primary_key=True) + ... name = sa.Column(sa.String) + ... code = sa.Column(sa.Integer) + +As seen below, the table doesn't exist yet: + + >>> engine.dialect.has_table(connection, table_name='departments') + False + +In order to create all missing tables, the ``create_all`` method can be used: + + >>> Base.metadata.create_all(bind=engine) + +With that, the table has been created: -The CrateDB Python client is available as package ``sqlalchemy-cratedb`` on `PyPI`_. + >>> engine.dialect.has_table(connection, table_name='departments') + True -To install the most recent driver version, including the SQLAlchemy dialect -extension, run:: +Let's also verify that by inquiring the ``information_schema.columns`` table: - pip install --upgrade sqlalchemy-cratedb + >>> stmt = ("select table_name, column_name, ordinal_position, data_type " + ... "from information_schema.columns " + ... "where table_name = 'departments' " + ... "order by column_name") + >>> pprint([str(r) for r in connection.execute(sa.text(stmt))]) + ["('departments', 'code', 3, 'integer')", + "('departments', 'id', 1, 'text')", + "('departments', 'name', 2, 'text')"] -After that is done, you can import the library, like so: -.. code-block:: python +Drop tables +----------- - >>> from sqlalchemy_cratedb import CrateDialect +In order to delete all tables reference within the ORM schema, invoke +``Base.metadata.drop_all()``. To delete a single table, use +``drop(...)``, as shown below: -Set up as a dependency -====================== + >>> Base.metadata.tables['departments'].drop(engine) -There are `many ways`_ to add the ``sqlalchemy-cratedb`` package as a dependency to your -project. All of them work equally well. Please note that you may want to employ -package version pinning in order to keep the environment of your project stable -and reproducible, achieving `repeatable installations`_. + >>> engine.dialect.has_table(connection, table_name='departments') + False -Next steps -========== +.. hidden: Disconnect from database -Learn how to :ref:`connect to CrateDB `. + >>> session.close() + >>> connection.close() + >>> engine.dispose() -.. _sqlalchemy-cratedb: https://pypi.org/project/sqlalchemy-cratedb/ -.. _CrateDB: https://crate.io/products/cratedb/ -.. _many ways: https://packaging.python.org/key_projects/ -.. _PyPI: https://pypi.org/ -.. _repeatable installations: https://pip.pypa.io/en/latest/topics/repeatable-installs/ +.. _URL: https://en.wikipedia.org/wiki/Uniform_Resource_Locator diff --git a/docs/index-all.rst b/docs/index-all.rst deleted file mode 100644 index 8ef1b682..00000000 --- a/docs/index-all.rst +++ /dev/null @@ -1,18 +0,0 @@ -:orphan: - -.. _index-all: - -################################## -CrateDB Python Client -- all pages -################################## - - -.. rubric:: Table of contents - -.. toctree:: - :maxdepth: 2 - - getting-started - sqlalchemy - data-types - by-example/index diff --git a/docs/index.rst b/docs/index.rst index 7cf2a23f..864dd6e8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,7 +38,7 @@ referenced below. .. toctree:: :titlesonly: - getting-started + install SQLAlchemy ========== @@ -56,7 +56,7 @@ supporting different kinds of `GeoJSON geometry objects`_. .. toctree:: :maxdepth: 2 - sqlalchemy + overview Install package from PyPI with DB API and SQLAlchemy support. @@ -123,12 +123,30 @@ please consult the :ref:`data-types` and :ref:`SQLAlchemy extension types data-types + +.. _examples: +.. _by-example: +.. _sqlalchemy-by-example: + Examples ======== -- The :ref:`by-example` section enumerates concise examples demonstrating the - different API interfaces of the CrateDB Python client library. Those are - DB API, HTTP, and BLOB interfaces, and the SQLAlchemy dialect. +This section enumerates concise examples demonstrating the +use of the SQLAlchemy dialect. + +.. toctree:: + :maxdepth: 1 + + getting-started + crud + working-with-types + advanced-querying + inspection-reflection + dataframe + + +See also +-------- - Executable code examples are maintained within the `cratedb-examples repository`_. - The `sample application`_ and the corresponding `sample application documentation`_ demonstrate the use of the driver on behalf of an example @@ -137,10 +155,6 @@ Examples connect to CrateDB using `pandas`_, and how to load and export data. - The `Apache Superset`_ and `FIWARE QuantumLeap data historian`_ projects. -.. toctree:: - :maxdepth: 2 - - by-example/index ******************* diff --git a/docs/by-example/sqlalchemy/inspection-reflection.rst b/docs/inspection-reflection.rst similarity index 94% rename from docs/by-example/sqlalchemy/inspection-reflection.rst rename to docs/inspection-reflection.rst index bb291157..db252216 100644 --- a/docs/by-example/sqlalchemy/inspection-reflection.rst +++ b/docs/inspection-reflection.rst @@ -1,8 +1,8 @@ -.. _sqlalchemy-inspection-reflection: +.. _inspection-reflection: -===================================================== -SQLAlchemy: Database schema inspection and reflection -===================================================== +========================================= +Database schema inspection and reflection +========================================= This section shows you how to inspect the schema of a database using CrateDB's SQLAlchemy integration. diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 00000000..dd4246c4 --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,52 @@ +.. _install: + +======= +Install +======= + +Learn how to install and get started with the Python client library for +`CrateDB`_. + +.. rubric:: Table of contents + +.. contents:: + :local: + +Install +======= + +.. highlight:: sh + +The CrateDB Python client is available as package ``sqlalchemy-cratedb`` on `PyPI`_. + +To install the most recent driver version, including the SQLAlchemy dialect +extension, run:: + + pip install --upgrade sqlalchemy-cratedb + +After that is done, you can import the library, like so: + +.. code-block:: python + + >>> from sqlalchemy_cratedb import CrateDialect + +Set up as a dependency +====================== + +There are `many ways`_ to add the ``sqlalchemy-cratedb`` package as a dependency to your +project. All of them work equally well. Please note that you may want to employ +package version pinning in order to keep the environment of your project stable +and reproducible, achieving `repeatable installations`_. + + +Next steps +========== + +Learn how to :ref:`connect to CrateDB `. + + +.. _sqlalchemy-cratedb: https://pypi.org/project/sqlalchemy-cratedb/ +.. _CrateDB: https://crate.io/products/cratedb/ +.. _many ways: https://packaging.python.org/key_projects/ +.. _PyPI: https://pypi.org/ +.. _repeatable installations: https://pip.pypa.io/en/latest/topics/repeatable-installs/ diff --git a/docs/sqlalchemy.rst b/docs/overview.rst similarity index 99% rename from docs/sqlalchemy.rst rename to docs/overview.rst index 8c399a5c..fb6a108a 100644 --- a/docs/sqlalchemy.rst +++ b/docs/overview.rst @@ -1,9 +1,9 @@ -.. _sqlalchemy-support: +.. _overview: .. _using-sqlalchemy: -================== -SQLAlchemy support -================== +======== +Overview +======== .. rubric:: Table of contents diff --git a/docs/by-example/sqlalchemy/working-with-types.rst b/docs/working-with-types.rst similarity index 97% rename from docs/by-example/sqlalchemy/working-with-types.rst rename to docs/working-with-types.rst index 169acede..2505f831 100644 --- a/docs/by-example/sqlalchemy/working-with-types.rst +++ b/docs/working-with-types.rst @@ -1,8 +1,8 @@ -.. _sqlalchemy-working-with-types: +.. _working-with-types: -============================================== -SQLAlchemy: Working with special CrateDB types -============================================== +================================== +Working with special CrateDB types +================================== This section of the documentation shows how to work with special data types from the CrateDB SQLAlchemy dialect. Currently, these are: diff --git a/pyproject.toml b/pyproject.toml index 81d166da..5bd43255 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -261,4 +261,5 @@ release = [ test = [ { cmd = "pytest" }, + { cmd = "python -m unittest -vvv tests/integration.py" }, ] diff --git a/src/crate/testing/testdata/mappings/locations.sql b/src/crate/testing/testdata/mappings/locations.sql deleted file mode 100644 index 3b2dabcd..00000000 --- a/src/crate/testing/testdata/mappings/locations.sql +++ /dev/null @@ -1,13 +0,0 @@ -create table locations ( - name string primary key, - date timestamp, - datetime_tz timestamp with time zone, - datetime_notz timestamp without time zone, - nullable_datetime timestamp, - nullable_date timestamp, - kind string, - flag boolean, - position integer, - description string, - details array(object) -) with (number_of_replicas=0) diff --git a/src/crate/testing/testdata/settings/test_a.json b/src/crate/testing/testdata/settings/test_a.json deleted file mode 100644 index 1dd23736..00000000 --- a/src/crate/testing/testdata/settings/test_a.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "index": { - "number_of_shards": 2, - "number_of_replicas": 0 - } -} diff --git a/src/crate/testing/testdata/data/test_a.json b/tests/assets/locations.jsonl similarity index 100% rename from src/crate/testing/testdata/data/test_a.json rename to tests/assets/locations.jsonl diff --git a/tests/assets/locations.sql b/tests/assets/locations.sql new file mode 100644 index 00000000..48597cd0 --- /dev/null +++ b/tests/assets/locations.sql @@ -0,0 +1,13 @@ +CREATE TABLE locations ( + name STRING PRIMARY KEY, + date TIMESTAMP, + datetime_tz TIMESTAMP WITH TIME ZONE, + datetime_notz TIMESTAMP WITHOUT TIME ZONE, + nullable_datetime TIMESTAMP, + nullable_date TIMESTAMP, + kind STRING, + flag BOOLEAN, + position INTEGER, + description STRING, + details ARRAY(OBJECT) +) WITH (number_of_replicas=0) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 00000000..99f21e88 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,14 @@ +# Docker Compose file for testing the SQLAlchemy dialect for CrateDB. +--- +# docker compose -f tests/docker-compose.yml up +version: "2.1" +services: + cratedb: + image: crate/crate:${CRATEDB_VERSION} + environment: + CRATE_HEAP_SIZE: 2g + volumes: + - ./assets:/assets + ports: + - "4200:4200" + - "5432:5432" diff --git a/src/crate/client/tests.py b/tests/integration.py similarity index 63% rename from src/crate/client/tests.py rename to tests/integration.py index a318d6de..71ae64ca 100644 --- a/src/crate/client/tests.py +++ b/tests/integration.py @@ -21,17 +21,16 @@ from __future__ import absolute_import +import os import sys import unittest import doctest from pprint import pprint import logging -from crate.testing.settings import crate_host, docs_path from crate.client import connect from sqlalchemy_cratedb import SA_VERSION, SA_1_4 - -makeSuite = unittest.TestLoader().loadTestsFromTestCase +from tests.settings import crate_host log = logging.getLogger() ch = logging.StreamHandler() @@ -45,33 +44,40 @@ def cprint(s): print(s) -crate_layer = None +def docs_path(*parts): + return os.path.abspath( + os.path.join( + os.path.dirname(os.path.dirname(__file__)), *parts + ) + ) -def setUpCrateLayerBaseline(test): - test.globs['crate_host'] = crate_host - test.globs['pprint'] = pprint - test.globs['print'] = cprint +def provision_database(): + + drop_tables() with connect(crate_host) as conn: cursor = conn.cursor() - with open(docs_path('testing/testdata/mappings/locations.sql')) as s: + with open(docs_path('tests/assets/locations.sql')) as s: stmt = s.read() cursor.execute(stmt) - stmt = ("select count(*) from information_schema.tables " - "where table_name = 'locations'") + stmt = ("SELECT COUNT(*) FROM information_schema.tables " + "WHERE table_name = 'locations'") cursor.execute(stmt) assert cursor.fetchall()[0][0] == 1 - data_path = docs_path('testing/testdata/data/test_a.json') + # `/assets` is located within the Docker container used for running CrateDB. + # docker compose -f tests/docker-compose.yml up + data_path = "/assets/locations.jsonl" + # load testing data into crate - cursor.execute("copy locations from ?", (data_path,)) + cursor.execute("COPY locations FROM ?", (data_path,)) # refresh location table so imported data is visible immediately - cursor.execute("refresh table locations") + cursor.execute("REFRESH TABLE locations") # create blob table - cursor.execute("create blob table myfiles clustered into 1 shards " + - "with (number_of_replicas=0)") + cursor.execute("CREATE BLOB TABLE myfiles CLUSTERED INTO 1 SHARDS " + + "WITH (number_of_replicas=0)") # create users cursor.execute("CREATE USER me WITH (password = 'my_secret_pw')") @@ -79,13 +85,6 @@ def setUpCrateLayerBaseline(test): cursor.close() - -def setUpCrateLayerSqlAlchemy(test): - """ - Setup tables and views needed for SQLAlchemy tests. - """ - setUpCrateLayerBaseline(test) - ddl_statements = [ """ CREATE TABLE characters ( @@ -94,8 +93,8 @@ def setUpCrateLayerSqlAlchemy(test): quote STRING, details OBJECT, more_details ARRAY(OBJECT), - INDEX name_ft USING fulltext(name) WITH (analyzer = 'english'), - INDEX quote_ft USING fulltext(quote) WITH (analyzer = 'english') + INDEX name_ft USING FULLTEXT(name) WITH (analyzer = 'english'), + INDEX quote_ft USING FULLTEXT(quote) WITH (analyzer = 'english') )""", """ CREATE VIEW characters_view @@ -108,36 +107,30 @@ def setUpCrateLayerSqlAlchemy(test): area GEO_SHAPE )""" ] - _execute_statements(ddl_statements, on_error="raise") - - -def tearDownDropEntitiesBaseline(test): - """ - Drop all tables, views, and users created by `setUpWithCrateLayer*`. - """ - ddl_statements = [ - "DROP TABLE locations", - "DROP BLOB TABLE myfiles", - "DROP USER me", - "DROP USER trusted_me", - ] _execute_statements(ddl_statements) -def tearDownDropEntitiesSqlAlchemy(test): +def drop_tables(): """ - Drop all tables, views, and users created by `setUpWithCrateLayer*`. + Drop all tables, views, and users created by the test suite. """ - tearDownDropEntitiesBaseline(test) ddl_statements = [ - "DROP TABLE characters", - "DROP VIEW characters_view", - "DROP TABLE cities", + "DROP TABLE IF EXISTS archived_tasks", + "DROP TABLE IF EXISTS characters", + "DROP TABLE IF EXISTS cities", + "DROP TABLE IF EXISTS locations", + "DROP BLOB TABLE IF EXISTS myfiles", + 'DROP TABLE IF EXISTS "test-testdrive"', + "DROP TABLE IF EXISTS todos", + 'DROP TABLE IF EXISTS "user"', + "DROP VIEW IF EXISTS characters_view", + "DROP USER IF EXISTS me", + "DROP USER IF EXISTS trusted_me", ] _execute_statements(ddl_statements) -def _execute_statements(statements, on_error="ignore"): +def _execute_statements(statements, on_error="raise"): with connect(crate_host) as conn: cursor = conn.cursor() for stmt in statements: @@ -145,7 +138,9 @@ def _execute_statements(statements, on_error="ignore"): cursor.close() -def _execute_statement(cursor, stmt, on_error="ignore"): +def _execute_statement(cursor, stmt, on_error="raise"): + if on_error not in ["ignore", "raise"]: + raise ValueError(f"Invalid value for `on_error` argument: {on_error}") try: cursor.execute(stmt) except Exception: # pragma: no cover @@ -159,33 +154,55 @@ def _execute_statement(cursor, stmt, on_error="ignore"): raise -def test_suite(): +def setUp(test): + + provision_database() + + test.globs['crate_host'] = crate_host + test.globs['pprint'] = pprint + test.globs['print'] = cprint + + +def tearDown(test): + pass + + +def create_test_suite(): suite = unittest.TestSuite() flags = (doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS) sqlalchemy_integration_tests = [ - 'docs/by-example/sqlalchemy/getting-started.rst', - 'docs/by-example/sqlalchemy/crud.rst', - 'docs/by-example/sqlalchemy/working-with-types.rst', - 'docs/by-example/sqlalchemy/advanced-querying.rst', - 'docs/by-example/sqlalchemy/inspection-reflection.rst', + 'docs/getting-started.rst', + 'docs/crud.rst', + 'docs/working-with-types.rst', + 'docs/advanced-querying.rst', + 'docs/inspection-reflection.rst', ] # Don't run DataFrame integration tests on SQLAlchemy 1.3 and Python 3.7. skip_dataframe = SA_VERSION < SA_1_4 or sys.version_info < (3, 8) if not skip_dataframe: sqlalchemy_integration_tests += [ - 'docs/by-example/sqlalchemy/dataframe.rst', + 'docs/dataframe.rst', ] s = doctest.DocFileSuite( *sqlalchemy_integration_tests, module_relative=False, - setUp=setUpCrateLayerSqlAlchemy, - tearDown=tearDownDropEntitiesSqlAlchemy, + setUp=setUp, + tearDown=tearDown, optionflags=flags, encoding='utf-8' ) suite.addTest(s) return suite + + +def load_tests(loader, tests, pattern): + """ + Provide test suite to test discovery. + + https://docs.python.org/3/library/unittest.html#load-tests-protocol + """ + return create_test_suite()