refactor: Use importlib.resources to load stream schemas (#125)

* refactor: Use importlib.resources to load stream schemas

* Fix poetry plugin name

* Run mypy with Nox

* Fix tests

.github/workflows/constraints.txt

@@ -3,3 +3,4 @@ nox==2023.4.22
 nox-poetry==1.0.3
 poetry==1.7.1
 poetry-dynamic-versioning==1.1.1
+poetry-plugin-export==1.6.0

.github/workflows/test.yml

@@ -17,14 +17,14 @@ jobs:
       FORCE_COLOR: "1"
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4.1.1
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.7.1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -36,6 +36,8 @@ jobs:
     - name: Install Poetry
       run: |
         pipx install poetry
+        pipx inject poetry poetry-plugin-export
+        pipx inject poetry poetry-dynamic-versioning
         poetry --version
 
     - name: Install Nox
@@ -44,9 +46,13 @@ jobs:
         pipx inject nox nox-poetry
         nox --version
 
-    - name: Run Nox
+    - name: Check Types
+      run: |
+        nox -s mypy-${{ matrix.python-version }}
+
+    - name: Run Tests
       env:
         TAP_BITSO_KEY: ${{ secrets.TAP_BITSO_KEY }}
         TAP_BITSO_SECRET: ${{ secrets.TAP_BITSO_SECRET }}
       run: |
-        nox
+        nox -s tests-${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -21,16 +21,8 @@ repos:
       - id: pyproject-fmt
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.1.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --show-fixes]
       - id: ruff-format
-
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
-    hooks:
-      - id: mypy
-        pass_filenames: true
-        additional_dependencies:
-          - types-requests

noxfile.py

@@ -21,8 +21,8 @@
 src_dir = "tap_bitso"
 tests_dir = "tests"
 
-python_versions = ["3.11", "3.10", "3.9", "3.8"]
-main_python_version = "3.10"
+python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8"]
+main_python_version = "3.11"
 locations = src_dir, tests_dir, "noxfile.py"
 nox.options.sessions = (
     "mypy",

pyproject.toml

@@ -23,9 +23,10 @@ repository = "https://github.com/edgarrmondragon/tap-bitso"
 documentation = "https://github.com/edgarrmondragon/tap-bitso#readme"
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.8"
+importlib-resources = {version = ">=6,<7", python = "<3.9"}
+python = "<4,>=3.8"
 singer-sdk = "~=0.34.0"
-structlog = ">=23.1,<23.3"
+structlog = ">=23,<24"
 
 [tool.poetry.dev-dependencies]
 colorama = ">=0.4.4"
@@ -65,6 +66,13 @@ unfixable = ["ERA001"]
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
+[tool.mypy]
+python_version = "3.11"
+warn_redundant_casts = true
+warn_unreachable = true
+warn_unused_configs = true
+warn_unused_ignores = true
+
 [[tool.mypy.overrides]]
 module = [
     "backoff",

tap_bitso/auth.py

@@ -13,7 +13,8 @@
 
 if TYPE_CHECKING:
     from requests import PreparedRequest
-    from singer_sdk.streams.core import Stream
+
+    from tap_bitso.streams import BitsoStream
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +25,7 @@ class BitsoAuthenticator(APIAuthenticatorBase):
     @classmethod
     def create_for_stream(
         cls: type[BitsoAuthenticator],
-        stream: Stream,
+        stream: BitsoStream,
     ) -> BitsoAuthenticator:
         """Create the authenticator for the stream.
 

tap_bitso/client.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator
 
 import backoff
@@ -15,8 +14,6 @@
 if TYPE_CHECKING:
     import requests
 
-SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
-
 
 class BitsoStream(RESTStream):
     """Bitso stream class."""

tap_bitso/schemas/__init__.py

@@ -0,0 +1,3 @@
+"""Stream schemas."""
+
+from __future__ import annotations

tap_bitso/streams.py

@@ -2,23 +2,32 @@
 
 from __future__ import annotations
 
+import sys
 import typing as t
-from pathlib import Path
 
+from tap_bitso import schemas
 from tap_bitso.client import BitsoStream
 
-SCHEMAS_DIR = Path(__file__).parent / "./schemas"
+if sys.version_info >= (3, 9):
+    from importlib import resources as importlib_resources
+else:
+    import importlib_resources
+
+SCHEMAS_DIR = importlib_resources.files(schemas)
 
 
 class LedgerStream(BitsoStream):
-    """Ledger stream."""
+    """Ledger stream.
+
+    DEPRECATED.
+    """
 
     name = "ledger"
     path = "/v3/ledger"
     replication_key = "eid"
     primary_keys: t.ClassVar[list[str]] = ["eid"]
     next_page_token_jsonpath = f"$.payload[-1].{replication_key}"
-    schema_filepath = SCHEMAS_DIR / "ledger.json"
+    schema_filepath = SCHEMAS_DIR / "ledger.json"  # type: ignore[assignment]
 
 
 class TradesStream(BitsoStream):
@@ -30,7 +39,7 @@ class TradesStream(BitsoStream):
     replication_key = "tid"
     primary_keys: t.ClassVar[list[str]] = ["tid"]
     next_page_token_jsonpath = f"$.payload[-1].{replication_key}"
-    schema_filepath = SCHEMAS_DIR / "trade.json"
+    schema_filepath = SCHEMAS_DIR / "trade.json"  # type: ignore[assignment]
 
 
 class UserTradesStream(BitsoStream):
@@ -42,7 +51,7 @@ class UserTradesStream(BitsoStream):
     replication_key = "tid"
     primary_keys: t.ClassVar[list[str]] = ["tid"]
     next_page_token_jsonpath = f"$.payload[-1].{replication_key}"
-    schema_filepath = SCHEMAS_DIR / "trade.json"
+    schema_filepath = SCHEMAS_DIR / "trade.json"  # type: ignore[assignment]
 
 
 class TickersStream(BitsoStream):
@@ -53,7 +62,7 @@ class TickersStream(BitsoStream):
     book_based = True
     records_jsonpath = "$.payload"
     primary_keys: t.ClassVar[list[str]] = ["book", "created_at"]
-    schema_filepath = SCHEMAS_DIR / "ticker.json"
+    schema_filepath = SCHEMAS_DIR / "ticker.json"  # type: ignore[assignment]
 
 
 class BooksStream(BitsoStream):
@@ -62,4 +71,4 @@ class BooksStream(BitsoStream):
     name = "books"
     path = "/v3/available_books"
     primary_keys: t.ClassVar[list[str]] = ["book"]
-    schema_filepath = SCHEMAS_DIR / "book.json"
+    schema_filepath = SCHEMAS_DIR / "book.json"  # type: ignore[assignment]

tap_bitso/tap.py

@@ -86,7 +86,7 @@ def discover_streams(self) -> list[Stream]:
         """
         return [
             streams.BooksStream(tap=self),
-            streams.LedgerStream(tap=self),
+            # streams.LedgerStream(tap=self),  # Removed upstream?  # noqa: ERA001
             streams.TickersStream(tap=self),
             streams.TradesStream(tap=self),
             streams.UserTradesStream(tap=self),

tests/conftest.py

@@ -0,0 +1,5 @@
+"""Pytest configuration for tests in this directory."""
+
+from __future__ import annotations
+
+pytest_plugins = ("singer_sdk.testing.pytest_plugin",)

tests/test_core.py

@@ -4,15 +4,18 @@
 
 from typing import Any
 
-from singer_sdk.testing import get_standard_tap_tests
+from singer_sdk.testing import SuiteConfig, get_tap_test_class
 
 from tap_bitso.tap import TapBitso
 
 SAMPLE_CONFIG: dict[str, Any] = {}
 
 
-def test_standard_tap_tests():
-    """Run standard tap tests from the SDK."""
-    tests = get_standard_tap_tests(TapBitso, config=SAMPLE_CONFIG)
-    for test in tests:
-        test()
+TestTapBitso = get_tap_test_class(
+    TapBitso,
+    config=SAMPLE_CONFIG,
+    suite_config=SuiteConfig(
+        ignore_no_records_for_streams=["user_trades"],
+        max_records_limit=50,
+    ),
+)