From dff37bd5fb5a557b30ba5340ed5683f49909f441 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 29 Aug 2024 21:06:32 +0200 Subject: [PATCH] drop Python 3.9 and add support for 3.12 (#502) --- .github/actions/setup-env/action.yml | 2 +- .github/workflows/test.yml | 20 ++-- .../workflows/update-docker-requirements.yml | 2 +- Dockerfile | 2 +- environment-dev.yml | 2 +- pyproject.toml | 10 +- ragna/_compat.py | 97 ------------------- ragna/assistants/_http_api.py | 3 +- ragna/core/_utils.py | 10 +- ragna/deploy/_api/core.py | 1 - ragna/deploy/_api/orm.py | 33 ++++++- ragna/deploy/_api/schemas.py | 18 +++- ragna/deploy/_ui/api_wrapper.py | 2 +- ragna/deploy/_ui/central_view.py | 2 - ragna/source_storages/_vector_database.py | 3 +- requirements-docker.lock | 4 +- tests/assistants/test_api.py | 1 - tests/test_dependencies.py | 6 +- 18 files changed, 77 insertions(+), 141 deletions(-) delete mode 100644 ragna/_compat.py diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 21a3742a..a7931388 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -3,7 +3,7 @@ description: "Setup development environment" inputs: python-version: - default: "3.9" + default: "3.10" description: "Python version to install." optional-dependencies: default: "true" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ff4147d..23ef589e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,12 +40,12 @@ jobs: - ubuntu-latest - windows-latest - macos-latest - python-version: ["3.9"] + python-version: ["3.10"] include: - - os: ubuntu-latest - python-version: "3.10" - os: ubuntu-latest python-version: "3.11" + - os: ubuntu-latest + python-version: "3.12" fail-fast: false @@ -91,22 +91,22 @@ jobs: - chromium - firefox python-version: - - "3.9" - "3.10" - - "3.11" + - "3.10" + - "3.12" exclude: - - python-version: "3.10" - os: windows-latest - python-version: "3.11" os: windows-latest - - python-version: "3.10" - os: macos-latest + - python-version: "3.12" + os: windows-latest - python-version: "3.11" os: macos-latest + - python-version: "3.12" + os: macos-latest include: - browser: webkit os: macos-latest - python-version: "3.9" + python-version: "3.10" fail-fast: false diff --git a/.github/workflows/update-docker-requirements.yml b/.github/workflows/update-docker-requirements.yml index 05845144..a1a2c860 100644 --- a/.github/workflows/update-docker-requirements.yml +++ b/.github/workflows/update-docker-requirements.yml @@ -30,7 +30,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-env with: - python-version: "3.11" + python-version: "3.12" optional-dependencies: "false" - name: Update docker requirements diff --git a/Dockerfile b/Dockerfile index de260634..735c877c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11 +FROM python:3.12 WORKDIR /opt/ragna diff --git a/environment-dev.yml b/environment-dev.yml index e42fe77d..523c71fb 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: ragna-dev channels: - conda-forge dependencies: - - python =3.9 + - python =3.10 - pip - git-lfs - pip: diff --git a/pyproject.toml b/pyproject.toml index ccdc84e9..0107939e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,12 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "aiofiles", "emoji", "fastapi", "httpx", - "importlib_metadata>=4.6; python_version<'3.10'", "packaging", "panel==1.4.4", "pydantic>=2", @@ -96,6 +95,7 @@ ignore = ["E501"] [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra --tb=short --asyncio-mode=auto" +asyncio_default_fixture_loop_scope = "function" testpaths = [ "tests", ] @@ -103,7 +103,11 @@ filterwarnings = [ "error", "ignore::ResourceWarning", # https://github.com/lancedb/lancedb/issues/1296 - "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" + "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning", + # chromadb -> opentelemetry -> google + "ignore:Type google._upb._message:DeprecationWarning", + # chromadb + "ignore:Python 3.14 will:DeprecationWarning" ] xfail_strict = true diff --git a/ragna/_compat.py b/ragna/_compat.py deleted file mode 100644 index c805a9fb..00000000 --- a/ragna/_compat.py +++ /dev/null @@ -1,97 +0,0 @@ -import builtins -import sys -from typing import ( - Any, - AsyncIterator, - Awaitable, - Callable, - Iterable, - Iterator, - Mapping, - TypeVar, - cast, -) - -__all__ = [ - "itertools_pairwise", - "importlib_metadata_package_distributions", - "aiter", - "anext", -] - -T = TypeVar("T") - - -def _itertools_pairwise() -> Callable[[Iterable[T]], Iterator[tuple[T, T]]]: - if sys.version_info[:2] >= (3, 10): - from itertools import pairwise - else: - from itertools import tee - from typing import Iterable, Iterator - - # https://docs.python.org/3/library/itertools.html#itertools.pairwise - def pairwise(iterable: Iterable[T]) -> Iterator[tuple[T, T]]: - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - return pairwise - - -itertools_pairwise = _itertools_pairwise() - - -def _importlib_metadata_package_distributions() -> ( - Callable[[], Mapping[str, list[str]]] -): - if sys.version_info[:2] >= (3, 10): - from importlib.metadata import packages_distributions - else: - from importlib_metadata import packages_distributions - - return packages_distributions - - -importlib_metadata_package_distributions = _importlib_metadata_package_distributions() - - -def _aiter() -> Callable[[Any], AsyncIterator]: - if sys.version_info[:2] >= (3, 10): - aiter = builtins.aiter - else: - - def aiter(obj: Any) -> AsyncIterator: - return cast(AsyncIterator, obj.__aiter__()) - - return aiter - - -aiter = _aiter() - - -def _anext() -> Callable[[AsyncIterator[T]], Awaitable[T]]: - if sys.version_info[:2] >= (3, 10): - anext = builtins.anext - else: - sentinel = object() - - def anext( - ait: AsyncIterator[T], - default: T = sentinel, # type: ignore[assignment] - ) -> Awaitable[T]: - async def anext_impl() -> T: - try: - return await ait.__anext__() - except StopAsyncIteration: - if default is not sentinel: - return default - - raise - - return anext_impl() - - return anext - - -anext = _anext() diff --git a/ragna/assistants/_http_api.py b/ragna/assistants/_http_api.py index d4a8a9a9..adc794b8 100644 --- a/ragna/assistants/_http_api.py +++ b/ragna/assistants/_http_api.py @@ -7,7 +7,6 @@ import httpx import ragna -from ragna._compat import anext from ragna.core import ( Assistant, EnvVarRequirement, @@ -132,7 +131,7 @@ async def read(self, n: int) -> bytes: # and set up decoding. if n == 0: return b"" - return await anext(self._ait, b"") # type: ignore[call-arg] + return await anext(self._ait, b"") @contextlib.asynccontextmanager async def _stream_json( diff --git a/ragna/core/_utils.py b/ragna/core/_utils.py index 897a69d6..972b0926 100644 --- a/ragna/core/_utils.py +++ b/ragna/core/_utils.py @@ -15,11 +15,7 @@ import pydantic import pydantic_core -from ragna._compat import importlib_metadata_package_distributions - -importlib_metadata_package_distributions = functools.cache( - importlib_metadata_package_distributions -) +packages_distributions = functools.cache(importlib.metadata.packages_distributions) class RagnaExceptionHttpDetail(enum.Enum): @@ -98,8 +94,8 @@ def is_available(self) -> bool: for module_name in { module_name - for module_name, distribution_names in importlib_metadata_package_distributions().items() - if distribution.name in distribution_names # type: ignore[attr-defined] + for module_name, distribution_names in packages_distributions().items() + if distribution.name in distribution_names and module_name not in self._exclude_modules }: try: diff --git a/ragna/deploy/_api/core.py b/ragna/deploy/_api/core.py index 78ee6154..77abea2f 100644 --- a/ragna/deploy/_api/core.py +++ b/ragna/deploy/_api/core.py @@ -19,7 +19,6 @@ import ragna import ragna.core -from ragna._compat import aiter, anext from ragna._utils import handle_localhost_origins from ragna.core import Assistant, Component, Rag, RagnaException, SourceStorage from ragna.core._rag import SpecialChatParams diff --git a/ragna/deploy/_api/orm.py b/ragna/deploy/_api/orm.py index 04a3583e..74f3560f 100644 --- a/ragna/deploy/_api/orm.py +++ b/ragna/deploy/_api/orm.py @@ -1,5 +1,6 @@ import json -from typing import Any +from datetime import datetime, timezone +from typing import Any, Optional from sqlalchemy import Column, ForeignKey, Table, types from sqlalchemy.engine import Dialect @@ -30,6 +31,34 @@ def process_result_value( return json.loads(value) +class UtcDateTime(types.TypeDecorator): + """UTC timezone aware datetime type. + + This is needed because sqlalchemy.types.DateTime(timezone=True) does not + consistently store the timezone. + """ + + impl = types.DateTime + + cache_ok = True + + def process_bind_param( # type: ignore[override] + self, value: Optional[datetime], dialect: Dialect + ) -> Optional[datetime]: + if value is not None: + assert value.tzinfo == timezone.utc + + return value + + def process_result_value( + self, value: Optional[datetime], dialect: Dialect + ) -> Optional[datetime]: + if value is None: + return None + + return value.replace(tzinfo=timezone.utc) + + class Base(DeclarativeBase): pass @@ -134,4 +163,4 @@ class Message(Base): secondary=source_message_association_table, back_populates="messages", ) - timestamp = Column(types.DateTime, nullable=False) + timestamp = Column(UtcDateTime, nullable=False) diff --git a/ragna/deploy/_api/schemas.py b/ragna/deploy/_api/schemas.py index 37471c69..cd6fb85b 100644 --- a/ragna/deploy/_api/schemas.py +++ b/ragna/deploy/_api/schemas.py @@ -1,14 +1,24 @@ from __future__ import annotations -import datetime import uuid -from typing import Any +from datetime import datetime, timezone +from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import AfterValidator, BaseModel, Field import ragna.core +def _set_utc_timezone(v: datetime) -> datetime: + if v.tzinfo is None: + return v.replace(tzinfo=timezone.utc) + else: + return v.astimezone(timezone.utc) + + +UtcDateTime = Annotated[datetime, AfterValidator(_set_utc_timezone)] + + class Components(BaseModel): documents: list[str] source_storages: list[dict[str, Any]] @@ -75,7 +85,7 @@ class Message(BaseModel): content: str role: ragna.core.MessageRole sources: list[Source] = Field(default_factory=list) - timestamp: datetime.datetime = Field(default_factory=datetime.datetime.utcnow) + timestamp: UtcDateTime = Field(default_factory=lambda: datetime.now(timezone.utc)) @classmethod def from_core(cls, message: ragna.core.Message) -> Message: diff --git a/ragna/deploy/_ui/api_wrapper.py b/ragna/deploy/_ui/api_wrapper.py index f96375de..5a655fd2 100644 --- a/ragna/deploy/_ui/api_wrapper.py +++ b/ragna/deploy/_ui/api_wrapper.py @@ -101,6 +101,6 @@ async def start_and_prepare( return chat["id"] def improve_message(self, msg): - msg["timestamp"] = datetime.strptime(msg["timestamp"], "%Y-%m-%dT%H:%M:%S.%f") + msg["timestamp"] = datetime.strptime(msg["timestamp"], "%Y-%m-%dT%H:%M:%S.%fZ") msg["content"] = emoji.emojize(msg["content"], language="alias") return msg diff --git a/ragna/deploy/_ui/central_view.py b/ragna/deploy/_ui/central_view.py index 173c6d55..c1b990c0 100644 --- a/ragna/deploy/_ui/central_view.py +++ b/ragna/deploy/_ui/central_view.py @@ -7,8 +7,6 @@ import param from panel.reactive import ReactiveHTML -from ragna._compat import anext - from . import styles as ui diff --git a/ragna/source_storages/_vector_database.py b/ragna/source_storages/_vector_database.py index b44c16da..81ec2df5 100644 --- a/ragna/source_storages/_vector_database.py +++ b/ragna/source_storages/_vector_database.py @@ -10,7 +10,6 @@ cast, ) -from ragna._compat import itertools_pairwise from ragna.core import ( PackageRequirement, Page, @@ -106,7 +105,7 @@ def _page_numbers_to_str(self, page_numbers: Optional[Iterable[int]]) -> str: ranges_str = [] range_int = [] - for current_page_number, next_page_number in itertools_pairwise( + for current_page_number, next_page_number in itertools.pairwise( itertools.chain(sorted(page_numbers), [None]) ): current_page_number = cast(int, current_page_number) diff --git a/requirements-docker.lock b/requirements-docker.lock index b954077d..0010d3c0 100644 --- a/requirements-docker.lock +++ b/requirements-docker.lock @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --extra=all --output-file=requirements-docker.lock --strip-extras pyproject.toml @@ -158,7 +158,7 @@ oauthlib==3.2.2 # via # kubernetes # requests-oauthlib -onnxruntime==1.16.3 +onnxruntime==1.19.0 # via chromadb opentelemetry-api==1.22.0 # via diff --git a/tests/assistants/test_api.py b/tests/assistants/test_api.py index 10ec9506..f7c9c594 100644 --- a/tests/assistants/test_api.py +++ b/tests/assistants/test_api.py @@ -9,7 +9,6 @@ import pytest from ragna import assistants -from ragna._compat import anext from ragna._utils import timeout_after from ragna.assistants._http_api import HttpApiAssistant, HttpStreamingProtocol from ragna.core import Message, RagnaException diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 03e62ca1..4fe56a0f 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -4,16 +4,16 @@ remove the offending test after you have cleaned up our code. """ -import pytest +from importlib.metadata import packages_distributions -from ragna._compat import importlib_metadata_package_distributions +import pytest @pytest.mark.xfail def test_pyarrow_dummy_module(): module_names = { module_name - for module_name, distribution_names in importlib_metadata_package_distributions().items() + for module_name, distribution_names in packages_distributions().items() if "pyarrow" in distribution_names } assert "__dummy__" not in module_names