chore: update Ruff to 0.8 (apify#738)

pyproject.toml

@@ -84,7 +84,7 @@ pytest-only = "~2.1.0"
 pytest-timeout = "~2.3.0"
 pytest-xdist = "~3.6.0"
 respx = "~0.21.0"
-ruff = "~0.7.0"
+ruff = "~0.8.0"
 setuptools = "~75.6.0"                    # setuptools are used by pytest, but not explicitly required
 sortedcontainers-stubs = "^2.4.2"
 types-beautifulsoup4 = "~4.12.0.20240229"
@@ -109,8 +109,6 @@ line-length = 120
 [tool.ruff.lint]
 select = ["ALL"]
 ignore = [
-    "ANN101",   # Missing type annotation for `self` in method
-    "ANN102",   # Missing type annotation for `{name}` in classmethod
     "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {filename}
     "ASYNC109", # Async function definition with a `timeout` parameter
     "BLE001",   # Do not catch blind exception
@@ -158,18 +156,21 @@ indent-style = "space"
     "TRY301",  # Abstract `raise` to an inner function
 ]
 "**/{docs}/**" = [
-    "D",       # Everything from the pydocstyle
-    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
-    "F841",    # Local variable {variable} is assigned to but never used
-    "N999",    # Invalid module name
+    "D",      # Everything from the pydocstyle
+    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
+    "F841",   # Local variable {variable} is assigned to but never used
+    "N999",   # Invalid module name
 ]
 
 [tool.ruff.lint.flake8-quotes]
 docstring-quotes = "double"
 inline-quotes = "single"
 
 [tool.ruff.lint.flake8-type-checking]
-runtime-evaluated-base-classes = ["pydantic.BaseModel", "pydantic_settings.BaseSettings"]
+runtime-evaluated-base-classes = [
+    "pydantic.BaseModel",
+    "pydantic_settings.BaseSettings",
+]
 
 [tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = ["id"]
@@ -217,7 +218,7 @@ ignore_missing_imports = true
 exclude_lines = [
     "pragma: no cover",
     "if TYPE_CHECKING:",
-    "assert_never()"
+    "assert_never()",
 ]
 
 [tool.basedpyright]

src/crawlee/_autoscaling/autoscaled_pool.py

@@ -7,13 +7,15 @@
 from contextlib import suppress
 from datetime import timedelta
 from logging import getLogger
-from typing import TYPE_CHECKING, Awaitable, Callable
+from typing import TYPE_CHECKING, Callable
 
 from crawlee._types import ConcurrencySettings
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 
 if TYPE_CHECKING:
+    from collections.abc import Awaitable
+
     from crawlee._autoscaling import SystemStatus
 
 logger = getLogger(__name__)

src/crawlee/_utils/lru_cache.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
 from collections import OrderedDict
-from collections.abc import MutableMapping
-from typing import Generic, ItemsView, Iterator, TypeVar, ValuesView
-from typing import OrderedDict as OrderedDictType
+from collections import OrderedDict as OrderedDictType
+from collections.abc import ItemsView, Iterator, MutableMapping, ValuesView
+from typing import Generic, TypeVar
 
 T = TypeVar('T')
 

src/crawlee/_utils/measure_time.py

@@ -3,7 +3,10 @@
 import time
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Iterator
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
 
 
 @dataclass

src/crawlee/base_storage_client/_base_dataset_client.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, AsyncContextManager, AsyncIterator
+from typing import TYPE_CHECKING
 
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+    from contextlib import AbstractAsyncContextManager
+
     from httpx import Response
 
     from crawlee._types import JsonSerializable
@@ -195,7 +198,7 @@ async def stream_items(
         skip_hidden: bool = False,
         xml_root: str | None = None,
         xml_row: str | None = None,
-    ) -> AsyncContextManager[Response | None]:
+    ) -> AbstractAsyncContextManager[Response | None]:
         """Retrieves dataset items as a streaming response.
 
         Args:

src/crawlee/base_storage_client/_base_key_value_store_client.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, AsyncContextManager
+from typing import TYPE_CHECKING, Any
 
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
+    from contextlib import AbstractAsyncContextManager
+
     from httpx import Response
 
     from crawlee.base_storage_client._models import (
@@ -90,7 +92,7 @@ async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord[bytes] | No
         """
 
     @abstractmethod
-    async def stream_record(self, key: str) -> AsyncContextManager[KeyValueStoreRecord[Response] | None]:
+    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
         """Retrieve the given record from the key-value store, as a stream.
 
         Args:

src/crawlee/basic_crawler/__init__.py

@@ -3,4 +3,4 @@
 from ._basic_crawler import BasicCrawler, BasicCrawlerOptions
 from ._context_pipeline import ContextPipeline
 
-__all__ = ['BasicCrawler', 'BasicCrawlingContext', 'BasicCrawlerOptions', 'ContextPipeline']
+__all__ = ['BasicCrawler', 'BasicCrawlerOptions', 'BasicCrawlingContext', 'ContextPipeline']

src/crawlee/basic_crawler/_basic_crawler.py

@@ -12,7 +12,7 @@
 from datetime import timedelta
 from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, AsyncContextManager, Callable, Generic, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Generic, Union, cast
 from urllib.parse import ParseResult, urlparse
 
 from tldextract import TLDExtract
@@ -47,6 +47,7 @@
 
 if TYPE_CHECKING:
     import re
+    from contextlib import AbstractAsyncContextManager
 
     from crawlee._types import ConcurrencySettings, HttpMethod, JsonSerializable
     from crawlee.base_storage_client._models import DatasetItemsListPage
@@ -130,7 +131,7 @@ class BasicCrawlerOptions(TypedDict, Generic[TCrawlingContext]):
     """Enables extending the request lifecycle and modifying the crawling context. Intended for use by
     subclasses rather than direct instantiation of `BasicCrawler`."""
 
-    _additional_context_managers: NotRequired[Sequence[AsyncContextManager]]
+    _additional_context_managers: NotRequired[Sequence[AbstractAsyncContextManager]]
     """Additional context managers used throughout the crawler lifecycle."""
 
     _logger: NotRequired[logging.Logger]
@@ -183,7 +184,7 @@ def __init__(
         configure_logging: bool = True,
         max_crawl_depth: int | None = None,
         _context_pipeline: ContextPipeline[TCrawlingContext] | None = None,
-        _additional_context_managers: Sequence[AsyncContextManager] | None = None,
+        _additional_context_managers: Sequence[AbstractAsyncContextManager] | None = None,
         _logger: logging.Logger | None = None,
     ) -> None:
         """A default constructor.

src/crawlee/basic_crawler/_context_pipeline.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import Any, AsyncGenerator, Awaitable, Callable, Generator, Generic, cast
+from collections.abc import AsyncGenerator, Awaitable, Generator
+from typing import Any, Callable, Generic, cast
 
 from typing_extensions import TypeVar
 

src/crawlee/beautifulsoup_crawler/_beautifulsoup_crawler.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import logging
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 from bs4 import BeautifulSoup, Tag
 from pydantic import ValidationError
@@ -19,6 +19,8 @@
 from crawlee.http_crawler import HttpCrawlingContext
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Iterable
+
     from typing_extensions import Unpack
 
     from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs

src/crawlee/errors.py

@@ -14,9 +14,9 @@
     'HttpStatusCodeError',
     'ProxyError',
     'RequestHandlerError',
+    'ServiceConflictError',
     'SessionError',
     'UserDefinedErrorHandlerError',
-    'ServiceConflictError',
 ]
 
 TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext, default=BasicCrawlingContext)

src/crawlee/http_crawler/_http_crawler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, AsyncGenerator, Iterable
+from typing import TYPE_CHECKING
 
 from crawlee._utils.docs import docs_group
 from crawlee.basic_crawler import BasicCrawler, BasicCrawlerOptions, ContextPipeline
@@ -10,6 +10,8 @@
 from crawlee.http_crawler._http_crawling_context import HttpCrawlingContext
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Iterable
+
     from typing_extensions import Unpack
 
     from crawlee._types import BasicCrawlingContext

src/crawlee/memory_storage_client/_dataset_client.py

@@ -6,7 +6,7 @@
 import shutil
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, Any, AsyncContextManager, AsyncIterator
+from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
 
@@ -19,6 +19,9 @@
 from crawlee.memory_storage_client._creation_management import find_or_create_client_by_id_or_name_inner
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+    from contextlib import AbstractAsyncContextManager
+
     from httpx import Response
 
     from crawlee._types import JsonSerializable
@@ -281,7 +284,7 @@ async def stream_items(
         skip_hidden: bool = False,
         xml_root: str | None = None,
         xml_row: str | None = None,
-    ) -> AsyncContextManager[Response | None]:
+    ) -> AbstractAsyncContextManager[Response | None]:
         raise NotImplementedError('This method is not supported in memory storage.')
 
     @override

src/crawlee/memory_storage_client/_key_value_store_client.py

@@ -6,7 +6,7 @@
 import shutil
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, Any, AsyncContextManager
+from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
 
@@ -28,6 +28,8 @@
 )
 
 if TYPE_CHECKING:
+    from contextlib import AbstractAsyncContextManager
+
     from httpx import Response
 
     from crawlee.memory_storage_client import MemoryStorageClient
@@ -218,7 +220,7 @@ async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord[bytes] | No
         return await self._get_record_internal(key, as_bytes=True)
 
     @override
-    async def stream_record(self, key: str) -> AsyncContextManager[KeyValueStoreRecord[Response] | None]:
+    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
         raise NotImplementedError('This method is not supported in memory storage.')
 
     @override

src/crawlee/parsel_crawler/_parsel_crawler.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import logging
-from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable
+from typing import TYPE_CHECKING, Any
 
 from parsel import Selector
 from pydantic import ValidationError
@@ -19,6 +19,8 @@
 from crawlee.parsel_crawler._parsel_crawling_context import ParselCrawlingContext
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator, Iterable
+
     from typing_extensions import Unpack
 
     from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs

src/crawlee/playwright_crawler/_playwright_crawler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Mapping
+from typing import TYPE_CHECKING, Any, Callable
 
 from pydantic import ValidationError
 
@@ -18,7 +18,7 @@
 from crawlee.playwright_crawler._utils import infinite_scroll
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import AsyncGenerator, Awaitable, Mapping
 
     from typing_extensions import Unpack
 

src/crawlee/playwright_crawler/_playwright_crawling_context.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Awaitable, Callable
+from typing import TYPE_CHECKING, Callable
 
 from crawlee._utils.docs import docs_group
 from crawlee.playwright_crawler._playwright_pre_navigation_context import PlaywrightPreNavigationContext
 
 if TYPE_CHECKING:
+    from collections.abc import Awaitable
+
     from playwright.async_api import Response
 
     from crawlee._types import EnqueueLinksFunction

src/crawlee/proxy_configuration.py

@@ -19,7 +19,7 @@
 
     from crawlee.base_storage_client._models import Request
 
-__all__ = ['ProxyInfo', 'ProxyConfiguration']
+__all__ = ['ProxyConfiguration', 'ProxyInfo']
 
 
 @dataclass

src/crawlee/router.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import Awaitable, Callable, Generic, TypeVar
+from collections.abc import Awaitable
+from typing import Callable, Generic, TypeVar
 
 from crawlee._types import BasicCrawlingContext
 from crawlee._utils.docs import docs_group

src/crawlee/service_container.py

@@ -15,15 +15,15 @@
     from crawlee.events._event_manager import EventManager
 
 __all__ = [
-    'get_storage_client',
-    'set_local_storage_client',
-    'set_cloud_storage_client',
-    'set_default_storage_client_type',
     'get_configuration',
     'get_configuration_if_set',
-    'set_configuration',
     'get_event_manager',
+    'get_storage_client',
+    'set_cloud_storage_client',
+    'set_configuration',
+    'set_default_storage_client_type',
     'set_event_manager',
+    'set_local_storage_client',
 ]
 
 StorageClientType = Literal['cloud', 'local']