From 10ea58f1b217865a1f84de6fa0e15405f7ac1459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Thu, 27 Apr 2023 21:44:40 -0600 Subject: [PATCH] Initial implementation of HTTP connector --- singer_sdk/authenticators.py | 50 ++++++++ singer_sdk/connectors/__init__.py | 3 +- singer_sdk/connectors/_http.py | 112 ++++++++++++++++++ singer_sdk/connectors/base.py | 69 +++++++++++ singer_sdk/connectors/sql.py | 37 +++--- singer_sdk/helpers/_compat.py | 6 +- tests/core/connectors/__init__.py | 0 tests/core/connectors/test_http_connector.py | 63 ++++++++++ .../test_sql_connector.py} | 0 9 files changed, 318 insertions(+), 22 deletions(-) create mode 100644 singer_sdk/connectors/_http.py create mode 100644 singer_sdk/connectors/base.py create mode 100644 tests/core/connectors/__init__.py create mode 100644 tests/core/connectors/test_http_connector.py rename tests/core/{test_connector_sql.py => connectors/test_sql_connector.py} (100%) diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index b379f6a879..1f8ed50a7e 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -13,6 +13,7 @@ import requests from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization +from requests.auth import AuthBase from singer_sdk.helpers._util import utc_now @@ -589,3 +590,52 @@ def oauth_request_payload(self) -> dict: "RS256", ), } + + +class NoopAuth(AuthBase): + """No-op authenticator.""" + + def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: + """Do nothing. + + Args: + r: The prepared request. + + Returns: + The unmodified prepared request. + """ + return r + + +class HeaderAuth(AuthBase): + """Header-based authenticator.""" + + def __init__( + self, + keyword: str, + value: str, + header: str = "Authorization", + ) -> None: + """Initialize the authenticator. + + Args: + keyword: The keyword to use in the header, e.g. "Bearer". + value: The value to use in the header, e.g. "my-token". + header: The header to add the keyword and value to, defaults to + ``"Authorization"``. + """ + self.keyword = keyword + self.value = value + self.header = header + + def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: + """Add the header to the request. + + Args: + r: The prepared request. + + Returns: + The prepared request with the header added. + """ + r.headers[self.header] = f"{self.keyword} {self.value}" + return r diff --git a/singer_sdk/connectors/__init__.py b/singer_sdk/connectors/__init__.py index 32799417a2..6e47df0871 100644 --- a/singer_sdk/connectors/__init__.py +++ b/singer_sdk/connectors/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +from ._http import HTTPConnector from .sql import SQLConnector -__all__ = ["SQLConnector"] +__all__ = ["HTTPConnector", "SQLConnector"] diff --git a/singer_sdk/connectors/_http.py b/singer_sdk/connectors/_http.py new file mode 100644 index 0000000000..d01faec576 --- /dev/null +++ b/singer_sdk/connectors/_http.py @@ -0,0 +1,112 @@ +"""HTTP-based tap class for Singer SDK.""" + +from __future__ import annotations + +import typing as t + +import requests + +from singer_sdk.authenticators import NoopAuth +from singer_sdk.connectors.base import BaseConnector + +if t.TYPE_CHECKING: + import sys + + from requests.adapters import BaseAdapter + + if sys.version_info >= (3, 10): + from typing import TypeAlias # noqa: ICN003 + else: + from typing_extensions import TypeAlias + +_Auth: TypeAlias = t.Callable[[requests.PreparedRequest], requests.PreparedRequest] + + +class HTTPConnector(BaseConnector[requests.Session]): + """Base class for all HTTP-based connectors.""" + + def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: + """Initialize the HTTP connector. + + Args: + config: Connector configuration parameters. + """ + super().__init__(config) + self._session = self.get_session() + self.refresh_auth() + + def get_connection(self, *, authenticate: bool = True) -> requests.Session: + """Return a new HTTP session object. + + Adds adapters and optionally authenticates the session. + + Args: + authenticate: Whether to authenticate the request. + + Returns: + A new HTTP session object. + """ + for prefix, adapter in self.adapters.items(): + self._session.mount(prefix, adapter) + + self._session.auth = self._auth if authenticate else None + + return self._session + + def get_session(self) -> requests.Session: + """Return a new HTTP session object. + + Returns: + A new HTTP session object. + """ + return requests.Session() + + def get_authenticator(self) -> _Auth: + """Authenticate the HTTP session. + + Returns: + An auth callable. + """ + return NoopAuth() + + def refresh_auth(self) -> None: + """Refresh the HTTP session authentication.""" + self._auth = self.get_authenticator() + + @property + def adapters(self) -> dict[str, BaseAdapter]: + """Return a mapping of URL prefixes to adapter objects. + + Returns: + A mapping of URL prefixes to adapter objects. + """ + return {} + + @property + def default_request_kwargs(self) -> dict[str, t.Any]: + """Return default kwargs for HTTP requests. + + Returns: + A mapping of default kwargs for HTTP requests. + """ + return {} + + def request( + self, + *args: t.Any, + authenticate: bool = True, + **kwargs: t.Any, + ) -> requests.Response: + """Make an HTTP request. + + Args: + *args: Positional arguments to pass to the request method. + authenticate: Whether to authenticate the request. + **kwargs: Keyword arguments to pass to the request method. + + Returns: + The HTTP response object. + """ + with self._connect(authenticate=authenticate) as session: + kwargs = {**self.default_request_kwargs, **kwargs} + return session.request(*args, **kwargs) diff --git a/singer_sdk/connectors/base.py b/singer_sdk/connectors/base.py new file mode 100644 index 0000000000..56d9540a96 --- /dev/null +++ b/singer_sdk/connectors/base.py @@ -0,0 +1,69 @@ +"""Base class for all connectors.""" + +from __future__ import annotations + +import abc +import typing as t +from contextlib import contextmanager + +from singer_sdk.helpers._compat import Protocol + +_T = t.TypeVar("_T", covariant=True) + + +class ContextManagerProtocol(Protocol[_T]): + """Protocol for context manager enter/exit.""" + + def __enter__(self) -> _T: # noqa: D105 + ... + + def __exit__(self, *args: t.Any) -> None: # noqa: D105 + ... + + +_C = t.TypeVar("_C", bound=ContextManagerProtocol) + + +class BaseConnector(abc.ABC, t.Generic[_C]): + """Base class for all connectors.""" + + def __init__(self, config: t.Mapping[str, t.Any] | None) -> None: + """Initialize the connector. + + Args: + config: Plugin configuration parameters. + """ + self._config = config or {} + + @property + def config(self) -> t.Mapping: + """Return the connector configuration. + + Returns: + A mapping of configuration parameters. + """ + return self._config + + @contextmanager + def _connect(self, *args: t.Any, **kwargs: t.Any) -> t.Generator[_C, None, None]: + """Connect to the destination. + + Args: + args: Positional arguments to pass to the connection method. + kwargs: Keyword arguments to pass to the connection method. + + Yields: + A connection object. + """ + with self.get_connection(*args, **kwargs) as connection: + yield connection + + @abc.abstractmethod + def get_connection(self, *args: t.Any, **kwargs: t.Any) -> _C: + """Connect to the destination. + + Args: + args: Positional arguments to pass to the connection method. + kwargs: Keyword arguments to pass to the connection method. + """ + ... diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 1af4c6f9b0..1cf607739f 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -5,7 +5,6 @@ import logging import typing as t import warnings -from contextlib import contextmanager from datetime import datetime from functools import lru_cache @@ -14,13 +13,14 @@ from singer_sdk import typing as th from singer_sdk._singerlib import CatalogEntry, MetadataMapping, Schema +from singer_sdk.connectors.base import BaseConnector from singer_sdk.exceptions import ConfigValidationError if t.TYPE_CHECKING: from sqlalchemy.engine.reflection import Inspector -class SQLConnector: +class SQLConnector(BaseConnector): """Base class for SQLAlchemy-based connectors. The connector class serves as a wrapper around the SQL connection. @@ -42,7 +42,7 @@ class SQLConnector: def __init__( self, - config: dict | None = None, + config: t.Mapping[str, t.Any] | None = None, sqlalchemy_url: str | None = None, ) -> None: """Initialize the SQL connector. @@ -51,18 +51,9 @@ def __init__( config: The parent tap or target object's config. sqlalchemy_url: Optional URL for the connection. """ - self._config: dict[str, t.Any] = config or {} + super().__init__(config=config) self._sqlalchemy_url: str | None = sqlalchemy_url or None - @property - def config(self) -> dict: - """If set, provides access to the tap or target config. - - Returns: - The settings as a dict. - """ - return self._config - @property def logger(self) -> logging.Logger: """Get logger. @@ -72,10 +63,20 @@ def logger(self) -> logging.Logger: """ return logging.getLogger("sqlconnector") - @contextmanager - def _connect(self) -> t.Iterator[sqlalchemy.engine.Connection]: - with self._engine.connect().execution_options(stream_results=True) as conn: - yield conn + def get_connection( + self, + *, + stream_results: bool = True, + ) -> sqlalchemy.engine.Connection: + """Return a new SQLAlchemy connection using the provided config. + + Args: + stream_results: Whether to stream results from the database. + + Returns: + A newly created SQLAlchemy connection object. + """ + return self._engine.connect().execution_options(stream_results=stream_results) def create_sqlalchemy_connection(self) -> sqlalchemy.engine.Connection: """(DEPRECATED) Return a new SQLAlchemy connection using the provided config. @@ -155,7 +156,7 @@ def sqlalchemy_url(self) -> str: return self._sqlalchemy_url - def get_sqlalchemy_url(self, config: dict[str, t.Any]) -> str: + def get_sqlalchemy_url(self, config: t.Mapping[str, t.Any]) -> str: """Return the SQLAlchemy URL string. Developers can generally override just one of the following: diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py index 87033ea4c5..8435eb87c9 100644 --- a/singer_sdk/helpers/_compat.py +++ b/singer_sdk/helpers/_compat.py @@ -6,9 +6,9 @@ if sys.version_info < (3, 8): import importlib_metadata as metadata - from typing_extensions import final + from typing_extensions import Protocol, final else: from importlib import metadata - from typing import final # noqa: ICN003 + from typing import Protocol, final # noqa: ICN003 -__all__ = ["metadata", "final"] +__all__ = ["metadata", "final", "Protocol"] diff --git a/tests/core/connectors/__init__.py b/tests/core/connectors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/connectors/test_http_connector.py b/tests/core/connectors/test_http_connector.py new file mode 100644 index 0000000000..15448ed04e --- /dev/null +++ b/tests/core/connectors/test_http_connector.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import typing as t + +from requests.adapters import BaseAdapter +from requests.models import PreparedRequest, Response + +from singer_sdk.authenticators import HeaderAuth +from singer_sdk.connectors import HTTPConnector + + +class MockAdapter(BaseAdapter): + def send( + self, + request: PreparedRequest, + stream: bool = False, # noqa: FBT002 + timeout: float | tuple[float, float] | tuple[float, None] | None = None, + verify: bool | str = True, # noqa: FBT002 + cert: bytes | str | tuple[bytes | str, bytes | str] | None = None, + proxies: t.Mapping[str, str] | None = None, + ) -> Response: + """Send a request.""" + response = Response() + data = { + "url": request.url, + "headers": dict(request.headers), + "method": request.method, + "body": request.body, + "stream": stream, + "timeout": timeout, + "verify": verify, + "cert": cert, + "proxies": proxies, + } + response.status_code = 200 + response._content = json.dumps(data).encode("utf-8") + return response + + def close(self) -> None: + pass + + +class MockConnector(HTTPConnector): + @property + def adapters(self) -> dict[str, BaseAdapter]: + return { + "https://test": MockAdapter(), + } + + def get_authenticator(self) -> HeaderAuth: + return HeaderAuth("Bearer", self.config["token"]) + + +def test_auth(): + connector = MockConnector({"token": "s3cr3t"}) + response = connector.request("GET", "https://test") + data = response.json() + assert data["headers"]["Authorization"] == "Bearer s3cr3t" + + response = connector.request("GET", "https://test", authenticate=False) + data = response.json() + assert "Authorization" not in data["headers"] diff --git a/tests/core/test_connector_sql.py b/tests/core/connectors/test_sql_connector.py similarity index 100% rename from tests/core/test_connector_sql.py rename to tests/core/connectors/test_sql_connector.py