From 002c4f80ab503f6c99bcb5c8e672960465814de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 27 Apr 2023 21:44:40 -0600 Subject: [PATCH 1/9] Initial implementation of HTTP connector --- .../singer_sdk.connectors.BaseConnector.rst | 8 ++ docs/guides/custom-connector.md | 32 +++++ docs/guides/index.md | 1 + docs/reference.rst | 9 ++ noxfile.py | 3 +- poetry.lock | 103 ++++++++++------ pyproject.toml | 1 + singer_sdk/authenticators.py | 50 ++++++++ singer_sdk/connectors/__init__.py | 4 +- singer_sdk/connectors/_http.py | 112 ++++++++++++++++++ singer_sdk/connectors/base.py | 69 +++++++++++ singer_sdk/connectors/sql.py | 37 +++--- singer_sdk/helpers/_compat.py | 6 +- singer_sdk/streams/rest.py | 25 +++- tests/core/connectors/__init__.py | 0 tests/core/connectors/test_http_connector.py | 105 ++++++++++++++++ .../test_sql_connector.py} | 0 17 files changed, 502 insertions(+), 63 deletions(-) create mode 100644 docs/classes/singer_sdk.connectors.BaseConnector.rst create mode 100644 docs/guides/custom-connector.md create mode 100644 singer_sdk/connectors/_http.py create mode 100644 singer_sdk/connectors/base.py create mode 100644 tests/core/connectors/__init__.py create mode 100644 tests/core/connectors/test_http_connector.py rename tests/core/{test_connector_sql.py => connectors/test_sql_connector.py} (100%) diff --git a/docs/classes/singer_sdk.connectors.BaseConnector.rst b/docs/classes/singer_sdk.connectors.BaseConnector.rst new file mode 100644 index 000000000..3ba703887 --- /dev/null +++ b/docs/classes/singer_sdk.connectors.BaseConnector.rst @@ -0,0 +1,8 @@ +singer_sdk.connectors.BaseConnector +=================================== + +.. currentmodule:: singer_sdk.connectors + +.. autoclass:: BaseConnector + :members: + :special-members: __init__, __call__ \ No newline at end of file diff --git a/docs/guides/custom-connector.md b/docs/guides/custom-connector.md new file mode 100644 index 000000000..cf0ec7e20 --- /dev/null +++ b/docs/guides/custom-connector.md @@ -0,0 +1,32 @@ +# Using a custom connector class + +The Singer SDK has a few built-in connector classes that are designed to work with a variety of sources: + +* [`SQLConnector`](../../classes/singer_sdk.SQLConnector) for SQL databases + +If you need to connect to a source that is not supported by one of these built-in connectors, you can create your own connector class. This guide will walk you through the process of creating a custom connector class. + +## Subclass `BaseConnector` + +The first step is to create a subclass of [`BaseConnector`](../../classes/singer_sdk.connectors.BaseConnector). This class is responsible for creating streams and handling the connection to the source. + +```python +from singer_sdk.connectors import BaseConnector + + +class MyConnector(BaseConnector): + pass +``` + +## Implement `get_connection` + +The [`get_connection`](http://127.0.0.1:5500/build/classes/singer_sdk.connectors.BaseConnector.html#singer_sdk.connectors.BaseConnector.get_connection) method is responsible for creating a connection to the source. It should return an object that implements the [context manager protocol](https://docs.python.org/3/reference/datamodel.html#with-statement-context-managers), e.g. it has `__enter__` and `__exit__` methods. + +```python +from singer_sdk.connectors import BaseConnector + + +class MyConnector(BaseConnector): + def get_connection(self): + return MyConnection() +``` diff --git a/docs/guides/index.md b/docs/guides/index.md index c4f5f8d69..2b6c07386 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -7,4 +7,5 @@ The following pages contain useful information for developers building on top of porting pagination-classes +custom-connector ``` diff --git a/docs/reference.rst b/docs/reference.rst index 0c8d8dff3..e288de55a 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -140,3 +140,12 @@ Batch batch.BaseBatcher batch.JSONLinesBatcher + +Abstract Connector Classes +-------------------------- + +.. autosummary:: + :toctree: classes + :template: class.rst + + connectors.BaseConnector diff --git a/noxfile.py b/noxfile.py index 4c4949413..45955c55d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -39,8 +39,9 @@ test_dependencies = [ "coverage[toml]", "pytest", - "pytest-snapshot", "pytest-durations", + "pytest-httpserver", + "pytest-snapshot", "freezegun", "pandas", "pyarrow", diff --git a/poetry.lock b/poetry.lock index ec835acb8..bb77cb0f6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -194,18 +194,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.129" +version = "1.26.138" description = "The AWS SDK for Python" category = "main" optional = true python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.129-py3-none-any.whl", hash = "sha256:1dab3fcbeada61a3b5a42ea25a89143511a5b22626775c685e31e313f327cf3c"}, - {file = "boto3-1.26.129.tar.gz", hash = "sha256:0686a62f424c4f3375a706555b765d1e24d03d70e7d317ffcb2d411b39aa8139"}, + {file = "boto3-1.26.138-py3-none-any.whl", hash = "sha256:d47a68a0ca6599e8711c7da670fbac24085d9d50cfb4f761204f154d2b6fae26"}, + {file = "boto3-1.26.138.tar.gz", hash = "sha256:f0a78f94a7140b60960898fd86677e4e73cc96bd7f3e5c64fc5cc1818d04c7b8"}, ] [package.dependencies] -botocore = ">=1.29.129,<1.30.0" +botocore = ">=1.29.138,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -214,14 +214,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.129" +version = "1.29.138" description = "Low-level, data-driven core of boto 3." category = "main" optional = true python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.129-py3-none-any.whl", hash = "sha256:f44460236324727615c518c229690c3cc3ea3162a55a2ac242ba771e8fa41553"}, - {file = "botocore-1.29.129.tar.gz", hash = "sha256:80370e835ccf12e0429d4c6cc0e9d03cf47b72c41ec5916b01fb9544765f314d"}, + {file = "botocore-1.29.138-py3-none-any.whl", hash = "sha256:3d145f30d10a9c712acee48e7ce906c9456bb25fe50d477c9312c702ccfa50d1"}, + {file = "botocore-1.29.138.tar.gz", hash = "sha256:31edc237088c104f7a05887646bbec31d7459dd2e108fd90cbffa315902817e2"}, ] [package.dependencies] @@ -448,14 +448,14 @@ files = [ [[package]] name = "commitizen" -version = "3.2.1" +version = "3.2.2" description = "Python commitizen client tool" category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "commitizen-3.2.1-py3-none-any.whl", hash = "sha256:3723a5bf612e75d9cd1b0499ca2902e96d5fa8aaaada3c3b8b5888039edd26a0"}, - {file = "commitizen-3.2.1.tar.gz", hash = "sha256:2d62fd099e81b5126d1870ada6c7e96ebb561cc27357bd2ce6930c0f5c573c21"}, + {file = "commitizen-3.2.2-py3-none-any.whl", hash = "sha256:1d967de9d1dc3210947fdbe280b34ac184d83dbe35661f21463cf0305fe670ef"}, + {file = "commitizen-3.2.2.tar.gz", hash = "sha256:62e06077e657ab6156baa8656a8d5e54db7c5c3f51feab6ea4d7b867ddeab325"}, ] [package.dependencies] @@ -490,7 +490,7 @@ PyGithub = "^1.57" type = "git" url = "https://github.com/meltano/commitizen-version-bump.git" reference = "main" -resolved_reference = "2ac24303b30441773d95357d5c2801275211ce5f" +resolved_reference = "98a4fc6437662037b5c618d80623b3b14ea8b4de" [[package]] name = "cookiecutter" @@ -1483,21 +1483,21 @@ files = [ [[package]] name = "platformdirs" -version = "3.5.0" +version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.5.0-py3-none-any.whl", hash = "sha256:47692bc24c1958e8b0f13dd727307cff1db103fca36399f457da8e05f222fdc4"}, - {file = "platformdirs-3.5.0.tar.gz", hash = "sha256:7954a68d0ba23558d753f73437c55f89027cf8f5108c19844d4b82e5af396335"}, + {file = "platformdirs-3.5.1-py3-none-any.whl", hash = "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5"}, + {file = "platformdirs-3.5.1.tar.gz", hash = "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f"}, ] [package.dependencies] typing-extensions = {version = ">=4.5", markers = "python_version < \"3.8\""} [package.extras] -docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.2.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] @@ -1641,14 +1641,14 @@ files = [ [[package]] name = "pygithub" -version = "1.58.1" +version = "1.58.2" description = "Use the full Github API v3" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "PyGithub-1.58.1-py3-none-any.whl", hash = "sha256:4e7fe9c3ec30d5fde5b4fbb97f18821c9dbf372bf6df337fe66f6689a65e0a83"}, - {file = "PyGithub-1.58.1.tar.gz", hash = "sha256:7d528b4ad92bc13122129fafd444ce3d04c47d2d801f6446b6e6ee2d410235b3"}, + {file = "PyGithub-1.58.2-py3-none-any.whl", hash = "sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8"}, + {file = "PyGithub-1.58.2.tar.gz", hash = "sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f"}, ] [package.dependencies] @@ -1797,6 +1797,21 @@ files = [ [package.dependencies] pytest = ">=4.6" +[[package]] +name = "pytest-httpserver" +version = "1.0.6" +description = "pytest-httpserver is a httpserver for pytest" +category = "dev" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pytest_httpserver-1.0.6-py3-none-any.whl", hash = "sha256:ac2379acc91fe8bdbe2911c93af8dd130e33b5899fb9934d15669480739c6d32"}, + {file = "pytest_httpserver-1.0.6.tar.gz", hash = "sha256:9040d07bf59ac45d8de3db1d4468fd2d1d607975e4da4c872ecc0402cdbf7b3e"}, +] + +[package.dependencies] +Werkzeug = ">=2.0.0" + [[package]] name = "pytest-snapshot" version = "0.9.0" @@ -2014,19 +2029,19 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "setuptools" -version = "67.7.2" +version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, + {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, + {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2415,7 +2430,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] @@ -2598,14 +2613,14 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.9" +version = "6.0.12.10" description = "Typing stubs for PyYAML" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, - {file = "types_PyYAML-6.0.12.9-py3-none-any.whl", hash = "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8"}, + {file = "types-PyYAML-6.0.12.10.tar.gz", hash = "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"}, + {file = "types_PyYAML-6.0.12.10-py3-none-any.whl", hash = "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f"}, ] [[package]] @@ -2637,14 +2652,14 @@ files = [ [[package]] name = "types-urllib3" -version = "1.26.25.12" +version = "1.26.25.13" description = "Typing stubs for urllib3" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-urllib3-1.26.25.12.tar.gz", hash = "sha256:a1557355ce8d350a555d142589f3001903757d2d36c18a66f588d9659bbc917d"}, - {file = "types_urllib3-1.26.25.12-py3-none-any.whl", hash = "sha256:3ba3d3a8ee46e0d5512c6bd0594da4f10b2584b47a470f8422044a2ab462f1df"}, + {file = "types-urllib3-1.26.25.13.tar.gz", hash = "sha256:3300538c9dc11dad32eae4827ac313f5d986b8b21494801f1bf97a1ac6c03ae5"}, + {file = "types_urllib3-1.26.25.13-py3-none-any.whl", hash = "sha256:5dbd1d2bef14efee43f5318b5d36d805a489f6600252bb53626d4bfafd95e27c"}, ] [[package]] @@ -2661,14 +2676,14 @@ files = [ [[package]] name = "urllib3" -version = "1.26.15" +version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, ] [package.extras] @@ -2688,6 +2703,24 @@ files = [ {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, ] +[[package]] +name = "werkzeug" +version = "2.2.3" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, + {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + [[package]] name = "wrapt" version = "1.15.0" @@ -2818,11 +2851,11 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -docs = ["sphinx", "furo", "sphinx-copybutton", "myst-parser", "sphinx-autobuild", "sphinx-reredirects"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-reredirects"] s3 = ["fs-s3fs"] testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7.1" -content-hash = "59c05b459c4b046dff2db9aa0629fd868b8865eb004cb48c9b7424bb1e7412d5" +content-hash = "dd0c7f4b737cdf19078e53b710c4b76071934c62dcb8c16b2385f796bfe2f09b" diff --git a/pyproject.toml b/pyproject.toml index 0eeb20e1f..fb439521e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ types-simplejson = "^3.18.0" types-PyYAML = "^6.0.12" coverage = {extras = ["toml"], version = "^7.2"} pyarrow = ">=11,<13" +pytest-httpserver = "^1.0.6" pytest-snapshot = "^0.9.0" # Cookiecutter tests diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index ac9bb2807..b1ee4d9cc 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -13,6 +13,7 @@ import requests from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization +from requests.auth import AuthBase from singer_sdk.helpers._util import utc_now @@ -590,3 +591,52 @@ def oauth_request_payload(self) -> dict: "RS256", ), } + + +class NoopAuth(AuthBase): + """No-op authenticator.""" + + def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: + """Do nothing. + + Args: + r: The prepared request. + + Returns: + The unmodified prepared request. + """ + return r + + +class HeaderAuth(AuthBase): + """Header-based authenticator.""" + + def __init__( + self, + keyword: str, + value: str, + header: str = "Authorization", + ) -> None: + """Initialize the authenticator. + + Args: + keyword: The keyword to use in the header, e.g. "Bearer". + value: The value to use in the header, e.g. "my-token". + header: The header to add the keyword and value to, defaults to + ``"Authorization"``. + """ + self.keyword = keyword + self.value = value + self.header = header + + def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: + """Add the header to the request. + + Args: + r: The prepared request. + + Returns: + The prepared request with the header added. + """ + r.headers[self.header] = f"{self.keyword} {self.value}" + return r diff --git a/singer_sdk/connectors/__init__.py b/singer_sdk/connectors/__init__.py index 32799417a..1c3916672 100644 --- a/singer_sdk/connectors/__init__.py +++ b/singer_sdk/connectors/__init__.py @@ -2,6 +2,8 @@ from __future__ import annotations +from ._http import HTTPConnector +from .base import BaseConnector from .sql import SQLConnector -__all__ = ["SQLConnector"] +__all__ = ["BaseConnector", "HTTPConnector", "SQLConnector"] diff --git a/singer_sdk/connectors/_http.py b/singer_sdk/connectors/_http.py new file mode 100644 index 000000000..d01faec57 --- /dev/null +++ b/singer_sdk/connectors/_http.py @@ -0,0 +1,112 @@ +"""HTTP-based tap class for Singer SDK.""" + +from __future__ import annotations + +import typing as t + +import requests + +from singer_sdk.authenticators import NoopAuth +from singer_sdk.connectors.base import BaseConnector + +if t.TYPE_CHECKING: + import sys + + from requests.adapters import BaseAdapter + + if sys.version_info >= (3, 10): + from typing import TypeAlias # noqa: ICN003 + else: + from typing_extensions import TypeAlias + +_Auth: TypeAlias = t.Callable[[requests.PreparedRequest], requests.PreparedRequest] + + +class HTTPConnector(BaseConnector[requests.Session]): + """Base class for all HTTP-based connectors.""" + + def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: + """Initialize the HTTP connector. + + Args: + config: Connector configuration parameters. + """ + super().__init__(config) + self._session = self.get_session() + self.refresh_auth() + + def get_connection(self, *, authenticate: bool = True) -> requests.Session: + """Return a new HTTP session object. + + Adds adapters and optionally authenticates the session. + + Args: + authenticate: Whether to authenticate the request. + + Returns: + A new HTTP session object. + """ + for prefix, adapter in self.adapters.items(): + self._session.mount(prefix, adapter) + + self._session.auth = self._auth if authenticate else None + + return self._session + + def get_session(self) -> requests.Session: + """Return a new HTTP session object. + + Returns: + A new HTTP session object. + """ + return requests.Session() + + def get_authenticator(self) -> _Auth: + """Authenticate the HTTP session. + + Returns: + An auth callable. + """ + return NoopAuth() + + def refresh_auth(self) -> None: + """Refresh the HTTP session authentication.""" + self._auth = self.get_authenticator() + + @property + def adapters(self) -> dict[str, BaseAdapter]: + """Return a mapping of URL prefixes to adapter objects. + + Returns: + A mapping of URL prefixes to adapter objects. + """ + return {} + + @property + def default_request_kwargs(self) -> dict[str, t.Any]: + """Return default kwargs for HTTP requests. + + Returns: + A mapping of default kwargs for HTTP requests. + """ + return {} + + def request( + self, + *args: t.Any, + authenticate: bool = True, + **kwargs: t.Any, + ) -> requests.Response: + """Make an HTTP request. + + Args: + *args: Positional arguments to pass to the request method. + authenticate: Whether to authenticate the request. + **kwargs: Keyword arguments to pass to the request method. + + Returns: + The HTTP response object. + """ + with self._connect(authenticate=authenticate) as session: + kwargs = {**self.default_request_kwargs, **kwargs} + return session.request(*args, **kwargs) diff --git a/singer_sdk/connectors/base.py b/singer_sdk/connectors/base.py new file mode 100644 index 000000000..b593a0e1c --- /dev/null +++ b/singer_sdk/connectors/base.py @@ -0,0 +1,69 @@ +"""Base class for all connectors.""" + +from __future__ import annotations + +import abc +import typing as t +from contextlib import contextmanager + +from singer_sdk.helpers._compat import Protocol + +_T = t.TypeVar("_T", covariant=True) + + +class ContextManagerProtocol(Protocol[_T]): + """Protocol for context manager enter/exit.""" + + def __enter__(self) -> _T: # noqa: D105 + ... # pragma: no cover + + def __exit__(self, *args: t.Any) -> None: # noqa: D105 + ... # pragma: no cover + + +_C = t.TypeVar("_C", bound=ContextManagerProtocol) + + +class BaseConnector(abc.ABC, t.Generic[_C]): + """Base class for all connectors.""" + + def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: + """Initialize the connector. + + Args: + config: Plugin configuration parameters. + """ + self._config = config or {} + + @property + def config(self) -> t.Mapping: + """Return the connector configuration. + + Returns: + A mapping of configuration parameters. + """ + return self._config + + @contextmanager + def _connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: + """Connect to the destination. + + Args: + args: Positional arguments to pass to the connection method. + kwargs: Keyword arguments to pass to the connection method. + + Yields: + A connection object. + """ + with self.get_connection(*args, **kwargs) as connection: + yield connection + + @abc.abstractmethod + def get_connection(self, *args: t.Any, **kwargs: t.Any) -> _C: + """Connect to the destination. + + Args: + args: Positional arguments to pass to the connection method. + kwargs: Keyword arguments to pass to the connection method. + """ + ... diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index aecfbb0c1..9033005f6 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -5,7 +5,6 @@ import logging import typing as t import warnings -from contextlib import contextmanager from datetime import datetime from functools import lru_cache @@ -14,13 +13,14 @@ from singer_sdk import typing as th from singer_sdk._singerlib import CatalogEntry, MetadataMapping, Schema +from singer_sdk.connectors.base import BaseConnector from singer_sdk.exceptions import ConfigValidationError if t.TYPE_CHECKING: from sqlalchemy.engine.reflection import Inspector -class SQLConnector: +class SQLConnector(BaseConnector): """Base class for SQLAlchemy-based connectors. The connector class serves as a wrapper around the SQL connection. @@ -42,7 +42,7 @@ class SQLConnector: def __init__( self, - config: dict | None = None, + config: t.Mapping[str, t.Any] | None = None, sqlalchemy_url: str | None = None, ) -> None: """Initialize the SQL connector. @@ -51,18 +51,9 @@ def __init__( config: The parent tap or target object's config. sqlalchemy_url: Optional URL for the connection. """ - self._config: dict[str, t.Any] = config or {} + super().__init__(config=config) self._sqlalchemy_url: str | None = sqlalchemy_url or None - @property - def config(self) -> dict: - """If set, provides access to the tap or target config. - - Returns: - The settings as a dict. - """ - return self._config - @property def logger(self) -> logging.Logger: """Get logger. @@ -72,10 +63,20 @@ def logger(self) -> logging.Logger: """ return logging.getLogger("sqlconnector") - @contextmanager - def _connect(self) -> t.Iterator[sqlalchemy.engine.Connection]: - with self._engine.connect().execution_options(stream_results=True) as conn: - yield conn + def get_connection( + self, + *, + stream_results: bool = True, + ) -> sqlalchemy.engine.Connection: + """Return a new SQLAlchemy connection using the provided config. + + Args: + stream_results: Whether to stream results from the database. + + Returns: + A newly created SQLAlchemy connection object. + """ + return self._engine.connect().execution_options(stream_results=stream_results) def create_sqlalchemy_connection(self) -> sqlalchemy.engine.Connection: """(DEPRECATED) Return a new SQLAlchemy connection using the provided config. @@ -155,7 +156,7 @@ def sqlalchemy_url(self) -> str: return self._sqlalchemy_url - def get_sqlalchemy_url(self, config: dict[str, t.Any]) -> str: + def get_sqlalchemy_url(self, config: t.Mapping[str, t.Any]) -> str: """Return the SQLAlchemy URL string. Developers can generally override just one of the following: diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py index 87033ea4c..8435eb87c 100644 --- a/singer_sdk/helpers/_compat.py +++ b/singer_sdk/helpers/_compat.py @@ -6,9 +6,9 @@ if sys.version_info < (3, 8): import importlib_metadata as metadata - from typing_extensions import final + from typing_extensions import Protocol, final else: from importlib import metadata - from typing import final # noqa: ICN003 + from typing import Protocol, final # noqa: ICN003 -__all__ = ["metadata", "final"] +__all__ = ["metadata", "final", "Protocol"] diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 61aa5e017..a3bdea60e 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -15,6 +15,7 @@ from singer_sdk import metrics from singer_sdk.authenticators import SimpleAuthenticator +from singer_sdk.connectors import HTTPConnector from singer_sdk.exceptions import FatalAPIError, RetriableAPIError from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.pagination import ( @@ -92,7 +93,13 @@ def __init__( if path: self.path = path self._http_headers: dict = {} - self._requests_session = requests.Session() + + self.connector = HTTPConnector(self.config) + + # Override the connector's auth with the stream's auth + self.connector._auth = self.authenticator + + self._requests_session = self.connector._session self._compiled_jsonpath = None self._next_page_token_compiled_jsonpath = None @@ -140,8 +147,14 @@ def requests_session(self) -> requests.Session: .. _requests.Session: https://requests.readthedocs.io/en/latest/api/#request-sessions """ + warn( + "The `requests_session` property is deprecated and will be removed in a " + "future release. Use the `connector` property instead.", + DeprecationWarning, + stacklevel=2, + ) if not self._requests_session: - self._requests_session = requests.Session() + self._requests_session = self.connector._session return self._requests_session def validate_response(self, response: requests.Response) -> None: @@ -262,7 +275,9 @@ def _request( Returns: TODO """ - response = self.requests_session.send(prepared_request, timeout=self.timeout) + with self.connector._connect() as session: + response = session.send(prepared_request, timeout=self.timeout) + self._write_request_duration_log( endpoint=self.path, response=response, @@ -333,8 +348,8 @@ def build_prepared_request( https://requests.readthedocs.io/en/latest/api/#requests.Request """ request = requests.Request(*args, **kwargs) - self.requests_session.auth = self.authenticator - return self.requests_session.prepare_request(request) + with self.connector._connect(authenticate=True) as session: + return session.prepare_request(request) def prepare_request( self, diff --git a/tests/core/connectors/__init__.py b/tests/core/connectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/connectors/test_http_connector.py b/tests/core/connectors/test_http_connector.py new file mode 100644 index 000000000..2c62ec94a --- /dev/null +++ b/tests/core/connectors/test_http_connector.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +import typing as t + +import requests +import werkzeug +from requests.adapters import BaseAdapter + +from singer_sdk.authenticators import HeaderAuth +from singer_sdk.connectors import HTTPConnector + +if t.TYPE_CHECKING: + from pytest_httpserver import HTTPServer + + +class MockAdapter(BaseAdapter): + def send( + self, + request: requests.PreparedRequest, + stream: bool = False, # noqa: FBT002 + timeout: float | tuple[float, float] | tuple[float, None] | None = None, + verify: bool | str = True, # noqa: FBT002 + cert: bytes | str | tuple[bytes | str, bytes | str] | None = None, + proxies: t.Mapping[str, str] | None = None, + ) -> requests.Response: + """Send a request.""" + response = requests.Response() + data = { + "url": request.url, + "headers": dict(request.headers), + "method": request.method, + "body": request.body, + "stream": stream, + "timeout": timeout, + "verify": verify, + "cert": cert, + "proxies": proxies, + } + response.status_code = 200 + response._content = json.dumps(data).encode("utf-8") + return response + + def close(self) -> None: + pass + + +class HeaderAuthConnector(HTTPConnector): + def get_authenticator(self) -> HeaderAuth: + return HeaderAuth("Bearer", self.config["token"]) + + +def test_base_connector(httpserver: HTTPServer): + connector = HTTPConnector({}) + + httpserver.expect_request("").respond_with_json({"foo": "bar"}) + url = httpserver.url_for("/") + + response = connector.request("GET", url) + data = response.json() + assert data["foo"] == "bar" + + +def test_auth(httpserver: HTTPServer): + connector = HeaderAuthConnector({"token": "s3cr3t"}) + + def _handler(request: werkzeug.Request) -> werkzeug.Response: + return werkzeug.Response( + json.dumps( + { + "headers": dict(request.headers), + "url": request.url, + }, + ), + status=200, + mimetype="application/json", + ) + + httpserver.expect_request("").respond_with_handler(_handler) + url = httpserver.url_for("/") + + response = connector.request("GET", url) + data = response.json() + assert data["headers"]["Authorization"] == "Bearer s3cr3t" + + response = connector.request("GET", url, authenticate=False) + data = response.json() + assert "Authorization" not in data["headers"] + + +def test_custom_adapters(): + class MyConnector(HTTPConnector): + @property + def adapters(self) -> dict[str, BaseAdapter]: + return { + "https://test": MockAdapter(), + } + + connector = MyConnector({}) + response = connector.request("GET", "https://test") + data = response.json() + + assert data["url"] == "https://test/" + assert data["headers"] + assert data["method"] == "GET" diff --git a/tests/core/test_connector_sql.py b/tests/core/connectors/test_sql_connector.py similarity index 100% rename from tests/core/test_connector_sql.py rename to tests/core/connectors/test_sql_connector.py From a40843172016c96198769f62522fd5ad31f59808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 11 Jul 2023 17:43:16 -0600 Subject: [PATCH 2/9] Feedback: Address private method and attribute access for connector API --- singer_sdk/connectors/_http.py | 39 +++++++++++++++++---- singer_sdk/connectors/base.py | 2 +- singer_sdk/connectors/sql.py | 24 ++++++++++--- singer_sdk/sinks/sql.py | 6 ++-- singer_sdk/streams/rest.py | 10 +++--- singer_sdk/streams/sql.py | 2 +- tests/core/connectors/test_sql_connector.py | 4 +-- 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/singer_sdk/connectors/_http.py b/singer_sdk/connectors/_http.py index d01faec57..935452490 100644 --- a/singer_sdk/connectors/_http.py +++ b/singer_sdk/connectors/_http.py @@ -32,7 +32,7 @@ def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: config: Connector configuration parameters. """ super().__init__(config) - self._session = self.get_session() + self.__session = self.get_session() self.refresh_auth() def get_connection(self, *, authenticate: bool = True) -> requests.Session: @@ -47,11 +47,11 @@ def get_connection(self, *, authenticate: bool = True) -> requests.Session: A new HTTP session object. """ for prefix, adapter in self.adapters.items(): - self._session.mount(prefix, adapter) + self.__session.mount(prefix, adapter) - self._session.auth = self._auth if authenticate else None + self.__session.auth = self.auth if authenticate else None - return self._session + return self.__session def get_session(self) -> requests.Session: """Return a new HTTP session object. @@ -71,7 +71,34 @@ def get_authenticator(self) -> _Auth: def refresh_auth(self) -> None: """Refresh the HTTP session authentication.""" - self._auth = self.get_authenticator() + self.auth = self.get_authenticator() + + @property + def auth(self) -> _Auth: + """Return the HTTP session authenticator. + + Returns: + An auth callable. + """ + return self.__auth + + @auth.setter + def auth(self, auth: _Auth) -> None: + """Set the HTTP session authenticator. + + Args: + auth: An auth callable. + """ + self.__auth = auth + + @property + def session(self) -> requests.Session: + """Return the HTTP session object. + + Returns: + The HTTP session object. + """ + return self.__session @property def adapters(self) -> dict[str, BaseAdapter]: @@ -107,6 +134,6 @@ def request( Returns: The HTTP response object. """ - with self._connect(authenticate=authenticate) as session: + with self.connect(authenticate=authenticate) as session: kwargs = {**self.default_request_kwargs, **kwargs} return session.request(*args, **kwargs) diff --git a/singer_sdk/connectors/base.py b/singer_sdk/connectors/base.py index b593a0e1c..ed5df69f5 100644 --- a/singer_sdk/connectors/base.py +++ b/singer_sdk/connectors/base.py @@ -45,7 +45,7 @@ def config(self) -> t.Mapping: return self._config @contextmanager - def _connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: + def connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: """Connect to the destination. Args: diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 9033005f6..92d279d9d 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -5,6 +5,7 @@ import logging import typing as t import warnings +from contextlib import contextmanager from datetime import datetime from functools import lru_cache @@ -63,6 +64,21 @@ def logger(self) -> logging.Logger: """ return logging.getLogger("sqlconnector") + @contextmanager + def _connect(self): # noqa: ANN202 + """Connect to the source. + + Yields: + A connection object. + """ + warnings.warn( + "`SQLConnector._connect` is deprecated. Use `SQLConnector.connect` instead.", + DeprecationWarning, + stacklevel=2, + ) + with self.connect() as connection: + yield connection + def get_connection( self, *, @@ -643,7 +659,7 @@ def create_schema(self, schema_name: str) -> None: Args: schema_name: The target schema to create. """ - with self._connect() as conn: + with self.connect() as conn: conn.execute(sqlalchemy.schema.CreateSchema(schema_name)) def create_empty_table( @@ -720,7 +736,7 @@ def _create_empty_column( column_name=column_name, column_type=sql_type, ) - with self._connect() as conn, conn.begin(): + with self.connect() as conn, conn.begin(): conn.execute(column_add_ddl) def prepare_schema(self, schema_name: str) -> None: @@ -814,7 +830,7 @@ def rename_column(self, full_table_name: str, old_name: str, new_name: str) -> N column_name=old_name, new_column_name=new_name, ) - with self._connect() as conn: + with self.connect() as conn: conn.execute(column_rename_ddl) def merge_sql_types( @@ -1123,5 +1139,5 @@ def _adapt_column_type( column_name=column_name, column_type=compatible_sql_type, ) - with self._connect() as conn: + with self.connect() as conn: conn.execute(alter_column_ddl) diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 6b6f8d121..6fca2db1a 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -328,7 +328,7 @@ def bulk_insert_records( else (self.conform_record(record) for record in records) ) self.logger.info("Inserting with SQL: %s", insert_sql) - with self.connector._connect() as conn, conn.begin(): + with self.connector.connect() as conn, conn.begin(): conn.execute(insert_sql, conformed_records) return len(conformed_records) if isinstance(conformed_records, list) else None @@ -379,7 +379,7 @@ def activate_version(self, new_version: int) -> None: ) if self.config.get("hard_delete", True): - with self.connector._connect() as conn, conn.begin(): + with self.connector.connect() as conn, conn.begin(): conn.execute( sqlalchemy.text( f"DELETE FROM {self.full_table_name} " # noqa: S608 @@ -408,7 +408,7 @@ def activate_version(self, new_version: int) -> None: bindparam("deletedate", value=deleted_at, type_=sqlalchemy.types.DateTime), bindparam("version", value=new_version, type_=sqlalchemy.types.Integer), ) - with self.connector._connect() as conn, conn.begin(): + with self.connector.connect() as conn, conn.begin(): conn.execute(query) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index e510e2a72..52682c253 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -99,9 +99,9 @@ def __init__( self.connector = HTTPConnector(self.config) # Override the connector's auth with the stream's auth - self.connector._auth = self.authenticator + self.connector.auth = self.authenticator - self._requests_session = self.connector._session + self._requests_session = self.connector.session self._compiled_jsonpath = None self._next_page_token_compiled_jsonpath = None @@ -156,7 +156,7 @@ def requests_session(self) -> requests.Session: stacklevel=2, ) if not self._requests_session: - self._requests_session = self.connector._session + self._requests_session = self.connector.session return self._requests_session def validate_response(self, response: requests.Response) -> None: @@ -277,7 +277,7 @@ def _request( Returns: TODO """ - with self.connector._connect() as session: + with self.connector.connect() as session: response = session.send(prepared_request, timeout=self.timeout) self._write_request_duration_log( @@ -350,7 +350,7 @@ def build_prepared_request( https://requests.readthedocs.io/en/latest/api/#requests.Request """ request = requests.Request(*args, **kwargs) - with self.connector._connect(authenticate=True) as session: + with self.connector.connect(authenticate=True) as session: return session.prepare_request(request) def prepare_request( diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py index d5fb52219..fc662a11a 100644 --- a/singer_sdk/streams/sql.py +++ b/singer_sdk/streams/sql.py @@ -202,7 +202,7 @@ def get_records(self, context: dict | None) -> t.Iterable[dict[str, t.Any]]: # processed. query = query.limit(self.ABORT_AT_RECORD_COUNT + 1) - with self.connector._connect() as conn: + with self.connector.connect() as conn: for record in conn.execute(query): transformed_record = self.post_process(dict(record._mapping)) if transformed_record is None: diff --git a/tests/core/connectors/test_sql_connector.py b/tests/core/connectors/test_sql_connector.py index 1c04dbcdd..b34c5ad4f 100644 --- a/tests/core/connectors/test_sql_connector.py +++ b/tests/core/connectors/test_sql_connector.py @@ -177,14 +177,14 @@ def test_connect_raises_on_operational_failure(self, connector): ) as _, connector._connect() as conn: conn.execute(sqlalchemy.text("SELECT * FROM fake_table")) - def test_rename_column_uses_connect_correctly(self, connector): + def test_rename_column_uses_connect_correctly(self, connector: SQLConnector): attached_engine = connector._engine # Ends up using the attached engine with mock.patch.object(attached_engine, "connect") as mock_conn: connector.rename_column("fake_table", "old_name", "new_name") mock_conn.assert_called_once() # Uses the _connect method - with mock.patch.object(connector, "_connect") as mock_connect_method: + with mock.patch.object(connector, "connect") as mock_connect_method: connector.rename_column("fake_table", "old_name", "new_name") mock_connect_method.assert_called_once() From 79c4f56fbbb7feee4c98542cc3c4826b8ad8b313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 11 Jul 2023 17:48:27 -0600 Subject: [PATCH 3/9] Fix warning line --- singer_sdk/connectors/sql.py | 6 ++---- tests/core/connectors/test_sql_connector.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 92d279d9d..3d301d0a1 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -72,7 +72,8 @@ def _connect(self): # noqa: ANN202 A connection object. """ warnings.warn( - "`SQLConnector._connect` is deprecated. Use `SQLConnector.connect` instead.", + "`SQLConnector._connect` is deprecated. " + "Use `SQLConnector.connect` instead.", DeprecationWarning, stacklevel=2, ) @@ -880,9 +881,6 @@ def merge_sql_types( if issubclass( generic_type, (sqlalchemy.types.String, sqlalchemy.types.Unicode), - ) or issubclass( - generic_type, - (sqlalchemy.types.String, sqlalchemy.types.Unicode), ): # If length None or 0 then is varchar max ? if ( diff --git a/tests/core/connectors/test_sql_connector.py b/tests/core/connectors/test_sql_connector.py index b34c5ad4f..574cc9ca1 100644 --- a/tests/core/connectors/test_sql_connector.py +++ b/tests/core/connectors/test_sql_connector.py @@ -148,13 +148,15 @@ def test_engine_creates_and_returns_cached_engine(self, connector): engine2 = connector._cached_engine assert engine1 is engine2 - def test_deprecated_functions_warn(self, connector): + def test_deprecated_functions_warn(self, connector: SQLConnector): with pytest.deprecated_call(): connector.create_sqlalchemy_engine() with pytest.deprecated_call(): connector.create_sqlalchemy_connection() with pytest.deprecated_call(): _ = connector.connection + with pytest.deprecated_call(), connector._connect() as _: + pass def test_connect_calls_engine(self, connector): with mock.patch.object( From 9045268e01c10110fa25698921456a172b2e6d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 11 Jul 2023 18:06:21 -0600 Subject: [PATCH 4/9] Test deprecations --- singer_sdk/streams/rest.py | 2 -- tests/core/test_streams.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 52682c253..9773dfc81 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -155,8 +155,6 @@ def requests_session(self) -> requests.Session: DeprecationWarning, stacklevel=2, ) - if not self._requests_session: - self._requests_session = self.connector.session return self._requests_session def validate_response(self, response: requests.Response) -> None: diff --git a/tests/core/test_streams.py b/tests/core/test_streams.py index 34bbc7514..fa9a5b5ae 100644 --- a/tests/core/test_streams.py +++ b/tests/core/test_streams.py @@ -594,3 +594,9 @@ def discover_streams(self): tap = MyTap(config=None, catalog=input_catalog) for stream in selection: assert tap.streams[stream].selected is selection[stream] + + +def test_deprecations(tap: SimpleTestTap): + stream = RestTestStream(tap=tap) + with pytest.deprecated_call(): + _ = stream.requests_session From 536bb2fa2246fd39f0a9c29873c0e64b9435af26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 11 Jul 2023 18:09:10 -0600 Subject: [PATCH 5/9] Fix session type annotation --- singer_sdk/streams/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 9773dfc81..4548e0e15 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -51,7 +51,7 @@ class RESTStream(Stream, t.Generic[_TToken], metaclass=abc.ABCMeta): """Abstract base class for REST API streams.""" _page_size: int = DEFAULT_PAGE_SIZE - _requests_session: requests.Session | None + _requests_session: requests.Session #: HTTP method to use for requests. Defaults to "GET". rest_method = "GET" From 138efbea12c8263cbca8b8e75e027d8ef32590e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 10:55:28 -0600 Subject: [PATCH 6/9] Rename type var --- singer_sdk/connectors/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/singer_sdk/connectors/base.py b/singer_sdk/connectors/base.py index ed5df69f5..e01846dc9 100644 --- a/singer_sdk/connectors/base.py +++ b/singer_sdk/connectors/base.py @@ -8,13 +8,13 @@ from singer_sdk.helpers._compat import Protocol -_T = t.TypeVar("_T", covariant=True) +_T_co = t.TypeVar("_T_co", covariant=True) -class ContextManagerProtocol(Protocol[_T]): +class ContextManagerProtocol(Protocol[_T_co]): """Protocol for context manager enter/exit.""" - def __enter__(self) -> _T: # noqa: D105 + def __enter__(self) -> _T_co: # noqa: D105 ... # pragma: no cover def __exit__(self, *args: t.Any) -> None: # noqa: D105 From 7fec03e0839edd08688b104154a8fecdd413a4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 27 Jul 2023 11:33:10 -0600 Subject: [PATCH 7/9] Fix types --- singer_sdk/connectors/base.py | 27 ++++++--------------------- singer_sdk/connectors/sql.py | 2 +- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/singer_sdk/connectors/base.py b/singer_sdk/connectors/base.py index e01846dc9..8e199b943 100644 --- a/singer_sdk/connectors/base.py +++ b/singer_sdk/connectors/base.py @@ -6,25 +6,11 @@ import typing as t from contextlib import contextmanager -from singer_sdk.helpers._compat import Protocol +_T = t.TypeVar("_T") -_T_co = t.TypeVar("_T_co", covariant=True) - -class ContextManagerProtocol(Protocol[_T_co]): - """Protocol for context manager enter/exit.""" - - def __enter__(self) -> _T_co: # noqa: D105 - ... # pragma: no cover - - def __exit__(self, *args: t.Any) -> None: # noqa: D105 - ... # pragma: no cover - - -_C = t.TypeVar("_C", bound=ContextManagerProtocol) - - -class BaseConnector(abc.ABC, t.Generic[_C]): +# class BaseConnector(abc.ABC, t.Generic[_T_co]): +class BaseConnector(abc.ABC, t.Generic[_T]): """Base class for all connectors.""" def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: @@ -45,7 +31,7 @@ def config(self) -> t.Mapping: return self._config @contextmanager - def connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: + def connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_T, None, None]: """Connect to the destination. Args: @@ -55,11 +41,10 @@ def connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: Yields: A connection object. """ - with self.get_connection(*args, **kwargs) as connection: - yield connection + yield self.get_connection(*args, **kwargs) @abc.abstractmethod - def get_connection(self, *args: t.Any, **kwargs: t.Any) -> _C: + def get_connection(self, *args: t.Any, **kwargs: t.Any) -> _T: """Connect to the destination. Args: diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 4cd25a732..c09c8951e 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -21,7 +21,7 @@ from sqlalchemy.engine.reflection import Inspector -class SQLConnector(BaseConnector): +class SQLConnector(BaseConnector[sqlalchemy.engine.Connection]): """Base class for SQLAlchemy-based connectors. The connector class serves as a wrapper around the SQL connection. From 0b35bde525fe14c243fb0a1ee6a3005d37633487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Wed, 29 May 2024 10:52:08 -0600 Subject: [PATCH 8/9] chore: Run `poetry lock` to install the latest transitive dependencies --- poetry.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0afc67626..e702a4f4b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -160,17 +160,17 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.34.110" +version = "1.34.114" description = "The AWS SDK for Python" optional = true python-versions = ">=3.8" files = [ - {file = "boto3-1.34.110-py3-none-any.whl", hash = "sha256:2fc871b4a5090716c7a71af52c462e539529227f4d4888fd04896d5028f9cedc"}, - {file = "boto3-1.34.110.tar.gz", hash = "sha256:83ffe2273da7bdfdb480d85b0705f04e95bd110e9741f23328b7c76c03e6d53c"}, + {file = "boto3-1.34.114-py3-none-any.whl", hash = "sha256:4460958d2b0c53bd2195b23ed5d45db2350e514486fe8caeb38b285b30742280"}, + {file = "boto3-1.34.114.tar.gz", hash = "sha256:eeb11bca9b19d12baf93436fb8a16b8b824f1f7e8b9bcc722607e862c46b1b08"}, ] [package.dependencies] -botocore = ">=1.34.110,<1.35.0" +botocore = ">=1.34.114,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -179,13 +179,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.110" +version = "1.34.114" description = "Low-level, data-driven core of boto 3." optional = true python-versions = ">=3.8" files = [ - {file = "botocore-1.34.110-py3-none-any.whl", hash = "sha256:1edf3a825ec0a5edf238b2d42ad23305de11d5a71bb27d6f9a58b7e8862df1b6"}, - {file = "botocore-1.34.110.tar.gz", hash = "sha256:b2c98c40ecf0b1facb9e61ceb7dfa28e61ae2456490554a16c8dbf99f20d6a18"}, + {file = "botocore-1.34.114-py3-none-any.whl", hash = "sha256:606d1e55984d45e41a812badee292755f4db0233eed9cca63ea3bb8f5755507f"}, + {file = "botocore-1.34.114.tar.gz", hash = "sha256:5705f74fda009656a218ffaf4afd81228359160f2ab806ab8222d07e9da3a73b"}, ] [package.dependencies] @@ -1613,13 +1613,13 @@ pytest = ">=4.6" [[package]] name = "pytest-httpserver" -version = "1.0.6" +version = "1.0.10" description = "pytest-httpserver is a httpserver for pytest" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8" files = [ - {file = "pytest_httpserver-1.0.6-py3-none-any.whl", hash = "sha256:ac2379acc91fe8bdbe2911c93af8dd130e33b5899fb9934d15669480739c6d32"}, - {file = "pytest_httpserver-1.0.6.tar.gz", hash = "sha256:9040d07bf59ac45d8de3db1d4468fd2d1d607975e4da4c872ecc0402cdbf7b3e"}, + {file = "pytest_httpserver-1.0.10-py3-none-any.whl", hash = "sha256:d40e0cc3d61ed6e4d80f52a796926d557a7db62b17e43b3e258a78a3c34becb9"}, + {file = "pytest_httpserver-1.0.10.tar.gz", hash = "sha256:77b9fbc2eb0a129cfbbacc8fe57e8cafe071d506489f31fe31e62f1b332d9905"}, ] [package.dependencies] @@ -1755,13 +1755,13 @@ rpds-py = ">=0.7.0" [[package]] name = "requests" -version = "2.32.2" +version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -2633,20 +2633,20 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "werkzeug" -version = "2.2.3" +version = "3.0.3" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, - {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, + {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, + {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, ] [package.dependencies] MarkupSafe = ">=2.1.1" [package.extras] -watchdog = ["watchdog"] +watchdog = ["watchdog (>=2.3)"] [[package]] name = "xdoctest" @@ -2673,13 +2673,13 @@ tests-strict = ["pytest (==4.6.0)", "pytest (==4.6.0)", "pytest (==6.2.5)", "pyt [[package]] name = "zipp" -version = "3.18.2" +version = "3.19.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.18.2-py3-none-any.whl", hash = "sha256:dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e"}, - {file = "zipp-3.18.2.tar.gz", hash = "sha256:6278d9ddbcfb1f1089a88fde84481528b07b0e10474e09dcfe53dad4069fa059"}, + {file = "zipp-3.19.0-py3-none-any.whl", hash = "sha256:96dc6ad62f1441bcaccef23b274ec471518daf4fbbc580341204936a5a3dddec"}, + {file = "zipp-3.19.0.tar.gz", hash = "sha256:952df858fb3164426c976d9338d3961e8e8b3758e2e059e0f754b8c4262625ee"}, ] [package.extras] From eedc63f7f0ccba95d96afa8085280da94251600e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Wed, 29 May 2024 10:54:01 -0600 Subject: [PATCH 9/9] Make Ruff happy --- singer_sdk/connectors/_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/singer_sdk/connectors/_http.py b/singer_sdk/connectors/_http.py index 935452490..8301233bb 100644 --- a/singer_sdk/connectors/_http.py +++ b/singer_sdk/connectors/_http.py @@ -53,7 +53,7 @@ def get_connection(self, *, authenticate: bool = True) -> requests.Session: return self.__session - def get_session(self) -> requests.Session: + def get_session(self) -> requests.Session: # noqa: PLR6301 """Return a new HTTP session object. Returns: @@ -61,7 +61,7 @@ def get_session(self) -> requests.Session: """ return requests.Session() - def get_authenticator(self) -> _Auth: + def get_authenticator(self) -> _Auth: # noqa: PLR6301 """Authenticate the HTTP session. Returns: