diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index 81b60ef270cb..a97950809068 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -8,9 +8,11 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration +from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader +from airbyte_cdk.sources.streams.checkpoint import Cursor from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.types import Config, StreamSlice @@ -157,3 +159,8 @@ def state_checkpoint_interval(self) -> Optional[int]: important state is the one at the beginning of the slice """ return None + + def get_cursor(self) -> Optional[Cursor]: + if self.retriever and isinstance(self.retriever, SimpleRetriever): + return self.retriever.cursor + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py index e5ac8478613a..3828c259b7fb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py @@ -5,5 +5,6 @@ from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import CursorFactory, PerPartitionCursor +from airbyte_cdk.sources.declarative.incremental.resumable_full_refresh_cursor import ResumableFullRefreshCursor -__all__ = ["CursorFactory", "DatetimeBasedCursor", "DeclarativeCursor", "PerPartitionCursor"] +__all__ = ["CursorFactory", "DatetimeBasedCursor", "DeclarativeCursor", "PerPartitionCursor", "ResumableFullRefreshCursor"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py index 052b32689000..baf794747a0b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py @@ -168,6 +168,11 @@ def stream_slices(self) -> Iterable[StreamSlice]: start_datetime = self._calculate_earliest_possible_value(self._select_best_end_datetime()) return self._partition_daterange(start_datetime, end_datetime, self._step) + def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]: + # Datetime based cursors operate over slices made up of datetime ranges. Stream state is based on the progress + # through each slice and does not belong to a specific slice. We just return stream state as it is. + return self.get_stream_state() + def _calculate_earliest_possible_value(self, end_datetime: datetime.datetime) -> datetime.datetime: lookback_delta = self._parse_timedelta(self._lookback_window.eval(self.config) if self._lookback_window else "P0D") earliest_possible_start_datetime = min(self._start_datetime.get_datetime(self.config), end_datetime) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py new file mode 100644 index 000000000000..05e36ad69a74 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py @@ -0,0 +1,96 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +from dataclasses import InitVar, dataclass +from typing import Any, Iterable, Mapping, Optional + +from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState + + +@dataclass +class ResumableFullRefreshCursor(DeclarativeCursor): + parameters: InitVar[Mapping[str, Any]] + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._cursor: StreamState = {} + + def get_stream_state(self) -> StreamState: + return self._cursor + + def set_initial_state(self, stream_state: StreamState) -> None: + self._cursor = stream_state + + def observe(self, stream_slice: StreamSlice, record: Record) -> None: + """ + Resumable full refresh manages state using a page number so it does not need to update state by observing incoming records. + """ + pass + + def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None: + # The ResumableFullRefreshCursor doesn't support nested streams yet so receiving a partition is unexpected + if stream_slice.partition: + raise ValueError(f"Stream slice {stream_slice} should not have a partition. Got {stream_slice.partition}.") + self._cursor = stream_slice.cursor_slice + + def should_be_synced(self, record: Record) -> bool: + """ + Unlike date-based cursors which filter out records outside slice boundaries, resumable full refresh records exist within pages + that don't have filterable bounds. We should always return them. + """ + return True + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + RFR record don't have ordering to be compared between one another. + """ + return False + + def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]: + # A top-level RFR cursor only manages the state of a single partition + return self._cursor + + def stream_slices(self) -> Iterable[StreamSlice]: + """ + Resumable full refresh cursors only return a single slice and can't perform partitioning because iteration is done per-page + along an unbounded set. + """ + yield from [StreamSlice(cursor_slice=self._cursor, partition={})] + + # This is an interesting pattern that might not seem obvious at first glance. This cursor itself has no functional need to + # inject any request values into the outbound response because the up-to-date pagination state is already loaded and + # maintained by the paginator component + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return {} + + def get_request_headers( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return {} + + def get_request_body_data( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return {} + + def get_request_body_json( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index bb0790f9ef45..5940673de590 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -28,7 +28,13 @@ from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector from airbyte_cdk.sources.declarative.extractors.record_selector import SCHEMA_TRANSFORMER_TYPE_MAPPING -from airbyte_cdk.sources.declarative.incremental import CursorFactory, DatetimeBasedCursor, DeclarativeCursor, PerPartitionCursor +from airbyte_cdk.sources.declarative.incremental import ( + CursorFactory, + DatetimeBasedCursor, + DeclarativeCursor, + PerPartitionCursor, + ResumableFullRefreshCursor, +) from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration @@ -668,6 +674,10 @@ def _merge_stream_slicers(self, model: DeclarativeStreamModel, config: Config) - ) elif model.incremental_sync: return self._create_component_from_model(model=model.incremental_sync, config=config) if model.incremental_sync else None + elif hasattr(model.retriever, "paginator") and model.retriever.paginator and not stream_slicer: + # To incrementally deliver RFR for low-code we're first implementing this for streams that do not use + # nested state like substreams or those using list partition routers + return ResumableFullRefreshCursor(parameters={}) elif stream_slicer: return stream_slicer else: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index ccfd91a1910f..d92de18120d7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -99,7 +99,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise ValueError("page_size_option cannot be set if the pagination strategy does not have a page_size") if isinstance(self.url_base, str): self.url_base = InterpolatedString(string=self.url_base, parameters=parameters) - self._token = self.pagination_strategy.initial_token + self._token: Optional[Any] = self.pagination_strategy.initial_token def next_page_token( self, response: requests.Response, last_page_size: int, last_record: Optional[Record] @@ -153,8 +153,11 @@ def get_request_body_json( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.body_json) - def reset(self) -> None: - self.pagination_strategy.reset() + def reset(self, reset_value: Optional[Any] = None) -> None: + if reset_value: + self.pagination_strategy.reset(reset_value=reset_value) + else: + self.pagination_strategy.reset() self._token = self.pagination_strategy.initial_token def _get_request_options(self, option_type: RequestOptionType) -> MutableMapping[str, Any]: @@ -235,6 +238,6 @@ def get_request_body_json( ) -> Mapping[str, Any]: return self._decorated.get_request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - def reset(self) -> None: + def reset(self, reset_value: Optional[Any] = None) -> None: self._decorated.reset() self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index 59fbcf6fcccf..4065902ffd09 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -60,6 +60,6 @@ def get_request_body_json( def next_page_token(self, response: requests.Response, last_page_size: int, last_record: Optional[Record]) -> Mapping[str, Any]: return {} - def reset(self) -> None: + def reset(self, reset_value: Optional[Any] = None) -> None: # No state to reset pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index 32752806f1dd..aebc8241a2d0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -21,7 +21,7 @@ class Paginator(ABC, RequestOptionsProvider): """ @abstractmethod - def reset(self) -> None: + def reset(self, reset_value: Optional[Any] = None) -> None: """ Reset the pagination's inner state """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index ac9143322cfb..fe576982b2a9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -35,6 +35,7 @@ class CursorPaginationStrategy(PaginationStrategy): decoder: Decoder = JsonDecoder(parameters={}) def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._initial_cursor = None if isinstance(self.cursor_value, str): self._cursor_value = InterpolatedString.create(self.cursor_value, parameters=parameters) else: @@ -46,7 +47,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: @property def initial_token(self) -> Optional[Any]: - return None + return self._initial_cursor def next_page_token(self, response: requests.Response, last_page_size: int, last_record: Optional[Record]) -> Optional[Any]: decoded_response = self.decoder.decode(response) @@ -74,9 +75,8 @@ def next_page_token(self, response: requests.Response, last_page_size: int, last ) return token if token else None - def reset(self) -> None: - # No state to reset - pass + def reset(self, reset_value: Optional[Any] = None) -> None: + self._initial_cursor = reset_value def get_page_size(self) -> Optional[int]: return self.page_size diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index 806df63e8f24..851d060b8f84 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -66,8 +66,11 @@ def next_page_token(self, response: requests.Response, last_page_size: int, last self._offset += last_page_size return self._offset - def reset(self) -> None: - self._offset = 0 + def reset(self, reset_value: Optional[Any] = 0) -> None: + if not isinstance(reset_value, int): + raise ValueError(f"Reset value {reset_value} for OffsetIncrement pagination strategy was not an integer") + else: + self._offset = reset_value def get_page_size(self) -> Optional[int]: if self._page_size: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index e56f86f4df4c..978ac1abaafd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -51,8 +51,13 @@ def next_page_token(self, response: requests.Response, last_page_size: int, last self._page += 1 return self._page - def reset(self) -> None: - self._page = self.start_from_page + def reset(self, reset_value: Optional[Any] = None) -> None: + if reset_value is None: + self._page = self.start_from_page + elif not isinstance(reset_value, int): + raise ValueError(f"Reset value {reset_value} for PageIncrement pagination strategy was not an integer") + else: + self._page = reset_value def get_page_size(self) -> Optional[int]: return self._page_size diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index 78232adfb4fb..135eb4812c6f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -34,7 +34,7 @@ def next_page_token(self, response: requests.Response, last_page_size: int, last pass @abstractmethod - def reset(self) -> None: + def reset(self, reset_value: Optional[Any] = None) -> None: """ Reset the pagination's inner state """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index ef1eb349b076..abc14f1cd827 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -1,15 +1,16 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import json from dataclasses import InitVar, dataclass, field from functools import partial from itertools import islice -from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union import requests from airbyte_cdk.models import AirbyteMessage from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector +from airbyte_cdk.sources.declarative.incremental import ResumableFullRefreshCursor from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter @@ -23,6 +24,8 @@ from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.utils.mapping_helpers import combine_mappings +FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" + @dataclass class SimpleRetriever(Retriever): @@ -69,6 +72,10 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._parameters = parameters self._name = InterpolatedString(self._name, parameters=parameters) if isinstance(self._name, str) else self._name + # This mapping is used during a resumable full refresh syncs to indicate whether a partition has started syncing + # records. Partitions serve as the key and map to True if they already began processing records + self._partition_started: MutableMapping[Any, bool] = dict() + @property # type: ignore def name(self) -> str: """ @@ -301,6 +308,26 @@ def _read_pages( # Always return an empty generator just in case no records were ever yielded yield from [] + def _read_page( + self, + records_generator_fn: Callable[[Optional[requests.Response]], Iterable[StreamData]], + stream_state: Mapping[str, Any], + stream_slice: StreamSlice, + ) -> Iterable[StreamData]: + response = self._fetch_next_page(stream_state, stream_slice) + yield from records_generator_fn(response) + + if not response: + next_page_token: Mapping[str, Any] = {FULL_REFRESH_SYNC_COMPLETE_KEY: True} + else: + next_page_token = self._next_page_token(response) or {FULL_REFRESH_SYNC_COMPLETE_KEY: True} + + if self.cursor: + self.cursor.close_slice(StreamSlice(cursor_slice=next_page_token, partition=stream_slice.partition)) + + # Always return an empty generator just in case no records were ever yielded + yield from [] + def read_records( self, records_schema: Mapping[str, Any], @@ -314,8 +341,6 @@ def read_records( :return: The records read from the API source """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check - # Fixing paginator types has a long tail of dependencies - self._paginator.reset() most_recent_record_from_slice = None record_generator = partial( @@ -324,19 +349,42 @@ def read_records( stream_slice=_slice, records_schema=records_schema, ) - for stream_data in self._read_pages(record_generator, self.state, _slice): - current_record = self._extract_record(stream_data, _slice) - if self.cursor and current_record: - self.cursor.observe(_slice, current_record) - # Latest record read, not necessarily within slice boundaries. - # TODO Remove once all custom components implement `observe` method. - # https://github.com/airbytehq/airbyte-internal-issues/issues/6955 - most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, current_record, _slice) - yield stream_data + if self.cursor and isinstance(self.cursor, ResumableFullRefreshCursor): + stream_state = self.state - if self.cursor: - self.cursor.close_slice(_slice, most_recent_record_from_slice) + # Before syncing the RFR stream, we check if the job's prior attempt was successful and don't need to fetch more records + # The platform deletes stream state for full refresh streams before starting a new job, so we don't need to worry about + # this value existing for the initial attempt + if stream_state.get(FULL_REFRESH_SYNC_COMPLETE_KEY): + return + cursor_value = stream_state.get("next_page_token") + + # The first attempt to read a page for the current partition should reset the paginator to the current + # cursor state which is initially assigned to the incoming state from the platform + partition_key = self._to_partition_key(_slice.partition) + if partition_key not in self._partition_started: + self._partition_started[partition_key] = True + self._paginator.reset(reset_value=cursor_value) + + yield from self._read_page(record_generator, stream_state, _slice) + else: + # Fixing paginator types has a long tail of dependencies + self._paginator.reset() + + for stream_data in self._read_pages(record_generator, self.state, _slice): + current_record = self._extract_record(stream_data, _slice) + if self.cursor and current_record: + self.cursor.observe(_slice, current_record) + + # Latest record read, not necessarily within slice boundaries. + # TODO Remove once all custom components implement `observe` method. + # https://github.com/airbytehq/airbyte-internal-issues/issues/6955 + most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, current_record, _slice) + yield stream_data + + if self.cursor: + self.cursor.close_slice(_slice, most_recent_record_from_slice) return def _get_most_recent_record( @@ -404,6 +452,11 @@ def _parse_records( def must_deduplicate_query_params(self) -> bool: return True + @staticmethod + def _to_partition_key(to_serialize: Any) -> str: + # separators have changed in Python 3.4. To avoid being impacted by further change, we explicitly specify our own value + return json.dumps(to_serialize, indent=None, separators=(",", ":"), sort_keys=True) + @dataclass class SimpleRetrieverTestReadDecorator(SimpleRetriever): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/__init__.py index 86a0673c923a..b0defda9c29b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/__init__.py @@ -1,7 +1,23 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -from .checkpoint_reader import CheckpointMode, CheckpointReader, FullRefreshCheckpointReader, IncrementalCheckpointReader, ResumableFullRefreshCheckpointReader +from .checkpoint_reader import ( + CheckpointMode, + CheckpointReader, + CursorBasedCheckpointReader, + FullRefreshCheckpointReader, + IncrementalCheckpointReader, + ResumableFullRefreshCheckpointReader +) from .cursor import Cursor -__all__ = ["CheckpointMode", "CheckpointReader", "Cursor", "FullRefreshCheckpointReader", "IncrementalCheckpointReader", "ResumableFullRefreshCheckpointReader"] + +__all__ = [ + "CheckpointMode", + "CheckpointReader", + "Cursor", + "CursorBasedCheckpointReader", + "FullRefreshCheckpointReader", + "IncrementalCheckpointReader", + "ResumableFullRefreshCheckpointReader" +] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py index acfe9c7ae39d..74ed9dd4984a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py @@ -4,6 +4,10 @@ from enum import Enum from typing import Any, Iterable, Mapping, Optional +from airbyte_cdk.sources.types import StreamSlice + +from .cursor import Cursor + class CheckpointMode(Enum): INCREMENTAL = "incremental" @@ -71,6 +75,83 @@ def get_checkpoint(self) -> Optional[Mapping[str, Any]]: return self._state +class CursorBasedCheckpointReader(CheckpointReader): + """ + CursorBasedCheckpointReader is used by streams that implement a Cursor in order to manage state. This allows the checkpoint + reader to delegate the complexity of fetching state to the cursor and focus on the iteration over a stream's partitions. + + This reader supports the Cursor interface used by Python and low-code sources. Not to be confused with Cursor interface + that belongs to the Concurrent CDK. + """ + + def __init__(self, cursor: Cursor, stream_slices: Iterable[Optional[Mapping[str, Any]]], read_state_from_cursor: bool = False): + self._cursor = cursor + self._stream_slices = iter(stream_slices) + # read_state_from_cursor is used to delineate that partitions should determine when to stop syncing dynamically according + # to the value of the state at runtime. This currently only applies to streams that use resumable full refresh. + self._read_state_from_cursor = read_state_from_cursor + self._current_slice: Optional[StreamSlice] = None + self._finished_sync = False + + def next(self) -> Optional[Mapping[str, Any]]: + """ + The next() method returns the next slice of data should be synced for the current stream according to its cursor. + This function support iterating over a stream's slices across two dimensions. The first dimension is the stream's + partitions like parent records for a substream. The inner dimension is iterating over the cursor value like a + date range for incremental streams or a pagination checkpoint for resumable full refresh. + + basic algorithm for iterating through a stream's slices is: + 1. The first time next() is invoked we get the first partition and return it + 2. For streams whose cursor value is determined dynamically using stream state + 1. Get the current state for the current partition + 2. If the current partition's state is complete, get the next partition + 3. If the current partition's state is still in progress, emit the next cursor value + 3. If a stream has processed all partitions, the iterator will raise a StopIteration exception signaling there are no more + slices left for extracting more records. + """ + + try: + if self._current_slice is None: + self._current_slice = self._get_next_slice() + return self._current_slice + if self._read_state_from_cursor: + state_for_slice = self._cursor.select_state(self._current_slice) + if state_for_slice == {"__ab_full_refresh_sync_complete": True}: + self._current_slice = self._get_next_slice() + else: + self._current_slice = StreamSlice(cursor_slice=state_for_slice or {}, partition=self._current_slice.partition) + else: + # Unlike RFR cursors that iterate dynamically based on how stream state is updated, most cursors operate on a + # fixed set of slices determined before reading records. They should just iterate to the next slice + self._current_slice = self._get_next_slice() + return self._current_slice + except StopIteration: + self._finished_sync = True + return None + + def _get_next_slice(self) -> StreamSlice: + next_slice = next(self._stream_slices) + if not isinstance(next_slice, StreamSlice): + raise ValueError( + f"{self._current_slice} should be of type StreamSlice. This is likely a bug in the CDK, please contact Airbyte support" + ) + return next_slice + + def observe(self, new_state: Mapping[str, Any]) -> None: + # Cursor based checkpoint readers don't need to observe the new state because it has already been updated by the cursor + # while processing records + pass + + def get_checkpoint(self) -> Optional[Mapping[str, Any]]: + # This is used to avoid sending a duplicate state message at the end of a sync since the stream has already + # emitted state at the end of each slice. We only emit state if _current_slice is None which indicates we had no + # slices and emitted no record or are currently in the process of emitting records. + if self._current_slice is None or not self._finished_sync: + return self._cursor.get_stream_state() + else: + return None + + class ResumableFullRefreshCheckpointReader(CheckpointReader): """ ResumableFullRefreshCheckpointReader allows for iteration over an unbounded set of records based on the pagination strategy diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/cursor.py index bf05b28c5fcd..4fad1364409d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/cursor.py @@ -3,7 +3,7 @@ # from abc import ABC, abstractmethod -from typing import Any +from typing import Any, Optional from airbyte_cdk.sources.types import Record, StreamSlice, StreamState @@ -67,3 +67,11 @@ def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: """ Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice """ + + @abstractmethod + def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]: + """ + Get the state value of a specific stream_slice. For incremental or resumable full refresh cursors which only manage state in + a single dimension this is the entire state object. For per-partition cursors used by substreams, this returns the state of + a specific parent delineated by the incoming slice's partition object. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py index 072893aacb6b..1f9f1210cda8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py @@ -15,6 +15,8 @@ from airbyte_cdk.sources.streams.checkpoint import ( CheckpointMode, CheckpointReader, + Cursor, + CursorBasedCheckpointReader, FullRefreshCheckpointReader, IncrementalCheckpointReader, ResumableFullRefreshCheckpointReader, @@ -235,8 +237,7 @@ def as_airbyte_stream(self) -> AirbyteStream: name=self.name, json_schema=dict(self.get_json_schema()), supported_sync_modes=[SyncMode.full_refresh], - # todo: This field doesn't exist yet, but it will in https://github.com/airbytehq/airbyte-protocol/pull/73 - # is_resumable=self.is_resumable, + is_resumable=self.is_resumable, ) if self.namespace: @@ -384,6 +385,14 @@ def get_updated_state( """ return {} + def get_cursor(self) -> Optional[Cursor]: + """ + A Cursor is an interface that a stream can implement to manage how its internal state is read and updated while + reading records. Historically, Python connectors had no concept of a cursor to manage state. Python streams need + need to define a cursor implementation and override this method to manage state through a Cursor. + """ + return None + def _get_checkpoint_reader( self, logger: logging.Logger, @@ -392,7 +401,19 @@ def _get_checkpoint_reader( stream_state: MutableMapping[str, Any], ) -> CheckpointReader: checkpoint_mode = self._checkpoint_mode + cursor = self.get_cursor() + if cursor: + slices = self.stream_slices( + cursor_field=cursor_field, + sync_mode=sync_mode, # todo: change this interface to no longer rely on sync_mode for behavior + stream_state=stream_state, + ) + return CursorBasedCheckpointReader( + stream_slices=slices, cursor=cursor, read_state_from_cursor=checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH + ) if checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH: + # Resumable full refresh readers rely on the stream state dynamically being updated during pagination and does + # not iterate over a static set of slices. return ResumableFullRefreshCheckpointReader(stream_state=stream_state) else: slices = self.stream_slices( diff --git a/airbyte-cdk/python/poetry.lock b/airbyte-cdk/python/poetry.lock index 55be1d2ffa45..f16087b314f2 100644 --- a/airbyte-cdk/python/poetry.lock +++ b/airbyte-cdk/python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -112,13 +112,13 @@ frozenlist = ">=1.1.0" [[package]] name = "airbyte-protocol-models" -version = "0.9.0" +version = "0.11.0" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.9.0-py3-none-any.whl", hash = "sha256:e972e140b5efd1edad5a338bcae8fdee9fc12545caf2c321e0f61b151c163a9b"}, - {file = "airbyte_protocol_models-0.9.0.tar.gz", hash = "sha256:40b69c33df23fe82d7078e84beb123bd604480e4d73cb277a890fcc92aedc8d2"}, + {file = "airbyte_protocol_models-0.11.0-py3-none-any.whl", hash = "sha256:2157757c1af8c13e471ab6a0304fd2f9a2a6af8cc9173937be1348a9553f7c32"}, + {file = "airbyte_protocol_models-0.11.0.tar.gz", hash = "sha256:1c7e46251b0d5a292b4aa382df24f415ac2a2a2b4719361b3c0f76368a043c23"}, ] [package.dependencies] @@ -896,17 +896,20 @@ idna = ">=2.0.0" [[package]] name = "emoji" -version = "2.11.1" +version = "2.12.1" description = "Emoji for Python" optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +python-versions = ">=3.7" files = [ - {file = "emoji-2.11.1-py2.py3-none-any.whl", hash = "sha256:b7ba25299bbf520cc8727848ae66b986da32aee27dc2887eaea2bff07226ce49"}, - {file = "emoji-2.11.1.tar.gz", hash = "sha256:062ff0b3154b6219143f8b9f4b3e5c64c35bc2b146e6e2349ab5f29e218ce1ee"}, + {file = "emoji-2.12.1-py3-none-any.whl", hash = "sha256:a00d62173bdadc2510967a381810101624a2f0986145b8da0cffa42e29430235"}, + {file = "emoji-2.12.1.tar.gz", hash = "sha256:4aa0488817691aa58d83764b6c209f8a27c0b3ab3f89d1b8dceca1a62e4973eb"}, ] +[package.dependencies] +typing-extensions = ">=4.7.0" + [package.extras] -dev = ["coverage", "coveralls", "pytest"] +dev = ["coverage", "pytest (>=7.4.4)"] [[package]] name = "et-xmlfile" @@ -1648,20 +1651,20 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langchain-text-splitters" -version = "0.0.1" +version = "0.0.2" description = "LangChain text splitting utilities" optional = true -python-versions = ">=3.8.1,<4.0" +python-versions = "<4.0,>=3.8.1" files = [ - {file = "langchain_text_splitters-0.0.1-py3-none-any.whl", hash = "sha256:f5b802f873f5ff6a8b9259ff34d53ed989666ef4e1582e6d1adb3b5520e3839a"}, - {file = "langchain_text_splitters-0.0.1.tar.gz", hash = "sha256:ac459fa98799f5117ad5425a9330b21961321e30bc19a2a2f9f761ddadd62aa1"}, + {file = "langchain_text_splitters-0.0.2-py3-none-any.whl", hash = "sha256:13887f32705862c1e1454213cb7834a63aae57c26fcd80346703a1d09c46168d"}, + {file = "langchain_text_splitters-0.0.2.tar.gz", hash = "sha256:ac8927dc0ba08eba702f6961c9ed7df7cead8de19a9f7101ab2b5ea34201b3c1"}, ] [package.dependencies] -langchain-core = ">=0.1.28,<0.2.0" +langchain-core = ">=0.1.28,<0.3" [package.extras] -extended-testing = ["lxml (>=5.1.0,<6.0.0)"] +extended-testing = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "lxml (>=4.9.3,<6.0)"] [[package]] name = "langdetect" @@ -1679,13 +1682,13 @@ six = "*" [[package]] name = "langsmith" -version = "0.1.57" +version = "0.1.60" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = true python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.57-py3-none-any.whl", hash = "sha256:dbd83b0944a2fbea4151f0aa053530d93fcf6784a580621bc60633cb890b57dc"}, - {file = "langsmith-0.1.57.tar.gz", hash = "sha256:4682204de19f0218029c2b8445ce2cc3485c8d0df9796b31e2ce4c9051fce365"}, + {file = "langsmith-0.1.60-py3-none-any.whl", hash = "sha256:3c3520ea473de0a984237b3e9d638fdf23ef3acc5aec89a42e693225e72d6120"}, + {file = "langsmith-0.1.60.tar.gz", hash = "sha256:6a145b5454437f9e0f81525f23c4dcdbb8c07b1c91553b8f697456c418d6a599"}, ] [package.dependencies] @@ -1955,39 +1958,40 @@ tests = ["pytest", "pytz", "simplejson"] [[package]] name = "matplotlib" -version = "3.8.4" +version = "3.9.0" description = "Python plotting package" optional = true python-versions = ">=3.9" files = [ - {file = "matplotlib-3.8.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:abc9d838f93583650c35eca41cfcec65b2e7cb50fd486da6f0c49b5e1ed23014"}, - {file = "matplotlib-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f65c9f002d281a6e904976007b2d46a1ee2bcea3a68a8c12dda24709ddc9106"}, - {file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1edd9f5383b504dbc26eeea404ed0a00656c526638129028b758fd43fc5f10"}, - {file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd79298550cba13a43c340581a3ec9c707bd895a6a061a78fa2524660482fc0"}, - {file = "matplotlib-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:90df07db7b599fe7035d2f74ab7e438b656528c68ba6bb59b7dc46af39ee48ef"}, - {file = "matplotlib-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:ac24233e8f2939ac4fd2919eed1e9c0871eac8057666070e94cbf0b33dd9c338"}, - {file = "matplotlib-3.8.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:72f9322712e4562e792b2961971891b9fbbb0e525011e09ea0d1f416c4645661"}, - {file = "matplotlib-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:232ce322bfd020a434caaffbd9a95333f7c2491e59cfc014041d95e38ab90d1c"}, - {file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6addbd5b488aedb7f9bc19f91cd87ea476206f45d7116fcfe3d31416702a82fa"}, - {file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc4ccdc64e3039fc303defd119658148f2349239871db72cd74e2eeaa9b80b71"}, - {file = "matplotlib-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b7a2a253d3b36d90c8993b4620183b55665a429da8357a4f621e78cd48b2b30b"}, - {file = "matplotlib-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:8080d5081a86e690d7688ffa542532e87f224c38a6ed71f8fbed34dd1d9fedae"}, - {file = "matplotlib-3.8.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6485ac1f2e84676cff22e693eaa4fbed50ef5dc37173ce1f023daef4687df616"}, - {file = "matplotlib-3.8.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c89ee9314ef48c72fe92ce55c4e95f2f39d70208f9f1d9db4e64079420d8d732"}, - {file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50bac6e4d77e4262c4340d7a985c30912054745ec99756ce213bfbc3cb3808eb"}, - {file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f51c4c869d4b60d769f7b4406eec39596648d9d70246428745a681c327a8ad30"}, - {file = "matplotlib-3.8.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b12ba985837e4899b762b81f5b2845bd1a28f4fdd1a126d9ace64e9c4eb2fb25"}, - {file = "matplotlib-3.8.4-cp312-cp312-win_amd64.whl", hash = "sha256:7a6769f58ce51791b4cb8b4d7642489df347697cd3e23d88266aaaee93b41d9a"}, - {file = "matplotlib-3.8.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:843cbde2f0946dadd8c5c11c6d91847abd18ec76859dc319362a0964493f0ba6"}, - {file = "matplotlib-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c13f041a7178f9780fb61cc3a2b10423d5e125480e4be51beaf62b172413b67"}, - {file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb44f53af0a62dc80bba4443d9b27f2fde6acfdac281d95bc872dc148a6509cc"}, - {file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:606e3b90897554c989b1e38a258c626d46c873523de432b1462f295db13de6f9"}, - {file = "matplotlib-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9bb0189011785ea794ee827b68777db3ca3f93f3e339ea4d920315a0e5a78d54"}, - {file = "matplotlib-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:6209e5c9aaccc056e63b547a8152661324404dd92340a6e479b3a7f24b42a5d0"}, - {file = "matplotlib-3.8.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7064120a59ce6f64103c9cefba8ffe6fba87f2c61d67c401186423c9a20fd35"}, - {file = "matplotlib-3.8.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0e47eda4eb2614300fc7bb4657fced3e83d6334d03da2173b09e447418d499f"}, - {file = "matplotlib-3.8.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:493e9f6aa5819156b58fce42b296ea31969f2aab71c5b680b4ea7a3cb5c07d94"}, - {file = "matplotlib-3.8.4.tar.gz", hash = "sha256:8aac397d5e9ec158960e31c381c5ffc52ddd52bd9a47717e2a694038167dffea"}, + {file = "matplotlib-3.9.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2bcee1dffaf60fe7656183ac2190bd630842ff87b3153afb3e384d966b57fe56"}, + {file = "matplotlib-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f988bafb0fa39d1074ddd5bacd958c853e11def40800c5824556eb630f94d3b"}, + {file = "matplotlib-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe428e191ea016bb278758c8ee82a8129c51d81d8c4bc0846c09e7e8e9057241"}, + {file = "matplotlib-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaf3978060a106fab40c328778b148f590e27f6fa3cd15a19d6892575bce387d"}, + {file = "matplotlib-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2e7f03e5cbbfacdd48c8ea394d365d91ee8f3cae7e6ec611409927b5ed997ee4"}, + {file = "matplotlib-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:13beb4840317d45ffd4183a778685e215939be7b08616f431c7795276e067463"}, + {file = "matplotlib-3.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:063af8587fceeac13b0936c42a2b6c732c2ab1c98d38abc3337e430e1ff75e38"}, + {file = "matplotlib-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a2fa6d899e17ddca6d6526cf6e7ba677738bf2a6a9590d702c277204a7c6152"}, + {file = "matplotlib-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550cdda3adbd596078cca7d13ed50b77879104e2e46392dcd7c75259d8f00e85"}, + {file = "matplotlib-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cce0f31b351e3551d1f3779420cf8f6ec0d4a8cf9c0237a3b549fd28eb4abb"}, + {file = "matplotlib-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c53aeb514ccbbcbab55a27f912d79ea30ab21ee0531ee2c09f13800efb272674"}, + {file = "matplotlib-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5be985db2596d761cdf0c2eaf52396f26e6a64ab46bd8cd810c48972349d1be"}, + {file = "matplotlib-3.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:c79f3a585f1368da6049318bdf1f85568d8d04b2e89fc24b7e02cc9b62017382"}, + {file = "matplotlib-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bdd1ecbe268eb3e7653e04f451635f0fb0f77f07fd070242b44c076c9106da84"}, + {file = "matplotlib-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d38e85a1a6d732f645f1403ce5e6727fd9418cd4574521d5803d3d94911038e5"}, + {file = "matplotlib-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a490715b3b9984fa609116481b22178348c1a220a4499cda79132000a79b4db"}, + {file = "matplotlib-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8146ce83cbc5dc71c223a74a1996d446cd35cfb6a04b683e1446b7e6c73603b7"}, + {file = "matplotlib-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:d91a4ffc587bacf5c4ce4ecfe4bcd23a4b675e76315f2866e588686cc97fccdf"}, + {file = "matplotlib-3.9.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:616fabf4981a3b3c5a15cd95eba359c8489c4e20e03717aea42866d8d0465956"}, + {file = "matplotlib-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd53c79fd02f1c1808d2cfc87dd3cf4dbc63c5244a58ee7944497107469c8d8a"}, + {file = "matplotlib-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06a478f0d67636554fa78558cfbcd7b9dba85b51f5c3b5a0c9be49010cf5f321"}, + {file = "matplotlib-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81c40af649d19c85f8073e25e5806926986806fa6d54be506fbf02aef47d5a89"}, + {file = "matplotlib-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52146fc3bd7813cc784562cb93a15788be0b2875c4655e2cc6ea646bfa30344b"}, + {file = "matplotlib-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:0fc51eaa5262553868461c083d9adadb11a6017315f3a757fc45ec6ec5f02888"}, + {file = "matplotlib-3.9.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bd4f2831168afac55b881db82a7730992aa41c4f007f1913465fb182d6fb20c0"}, + {file = "matplotlib-3.9.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:290d304e59be2b33ef5c2d768d0237f5bd132986bdcc66f80bc9bcc300066a03"}, + {file = "matplotlib-3.9.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff2e239c26be4f24bfa45860c20ffccd118d270c5b5d081fa4ea409b5469fcd"}, + {file = "matplotlib-3.9.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:af4001b7cae70f7eaacfb063db605280058246de590fa7874f00f62259f2df7e"}, + {file = "matplotlib-3.9.0.tar.gz", hash = "sha256:e6d29ea6c19e34b30fb7d88b7081f869a03014f66fe06d62cc77d5a6ea88ed7a"}, ] [package.dependencies] @@ -1996,12 +2000,15 @@ cycler = ">=0.10" fonttools = ">=4.22.0" importlib-resources = {version = ">=3.2.0", markers = "python_version < \"3.10\""} kiwisolver = ">=1.3.1" -numpy = ">=1.21" +numpy = ">=1.23" packaging = ">=20.0" pillow = ">=8" pyparsing = ">=2.3.1" python-dateutil = ">=2.7" +[package.extras] +dev = ["meson-python (>=0.13.1)", "numpy (>=1.25)", "pybind11 (>=2.6)", "setuptools (>=64)", "setuptools_scm (>=7)"] + [[package]] name = "mccabe" version = "0.7.0" @@ -2461,13 +2468,13 @@ xml = ["lxml (>=4.6.3)"] [[package]] name = "pandas-stubs" -version = "2.2.1.240316" +version = "2.2.2.240514" description = "Type annotations for pandas" optional = true python-versions = ">=3.9" files = [ - {file = "pandas_stubs-2.2.1.240316-py3-none-any.whl", hash = "sha256:0126a26451a37cb893ea62357ca87ba3d181bd999ec8ba2ca5602e20207d6682"}, - {file = "pandas_stubs-2.2.1.240316.tar.gz", hash = "sha256:236a4f812fb6b1922e9607ff09e427f6d8540c421c9e5a40e3e4ddf7adac7f05"}, + {file = "pandas_stubs-2.2.2.240514-py3-none-any.whl", hash = "sha256:5d6f64d45a98bc94152a0f76fa648e598cd2b9ba72302fd34602479f0c391a53"}, + {file = "pandas_stubs-2.2.2.240514.tar.gz", hash = "sha256:85b20da44a62c80eb8389bcf4cbfe31cce1cafa8cca4bf1fc75ec45892e72ce8"}, ] [package.dependencies] @@ -2652,13 +2659,13 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.2.1" +version = "4.2.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, - {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] [package.extras] @@ -3256,101 +3263,104 @@ files = [ [[package]] name = "rapidfuzz" -version = "3.9.0" +version = "3.9.1" description = "rapid fuzzy string matching" optional = true python-versions = ">=3.8" files = [ - {file = "rapidfuzz-3.9.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bd375c4830fee11d502dd93ecadef63c137ae88e1aaa29cc15031fa66d1e0abb"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55e2c5076f38fc1dbaacb95fa026a3e409eee6ea5ac4016d44fb30e4cad42b20"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:488f74126904db6b1bea545c2f3567ea882099f4c13f46012fe8f4b990c683df"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3f2d1ea7cd57dfcd34821e38b4924c80a31bcf8067201b1ab07386996a9faee"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b11e602987bcb4ea22b44178851f27406fca59b0836298d0beb009b504dba266"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3083512e9bf6ed2bb3d25883922974f55e21ae7f8e9f4e298634691ae1aee583"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b33c6d4b3a1190bc0b6c158c3981535f9434e8ed9ffa40cf5586d66c1819fb4b"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dcb95fde22f98e6d0480db8d6038c45fe2d18a338690e6f9bba9b82323f3469"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:08d8b49b3a4fb8572e480e73fcddc750da9cbb8696752ee12cca4bf8c8220d52"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e721842e6b601ebbeb8cc5e12c75bbdd1d9e9561ea932f2f844c418c31256e82"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7988363b3a415c5194ce1a68d380629247f8713e669ad81db7548eb156c4f365"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2d267d4c982ab7d177e994ab1f31b98ff3814f6791b90d35dda38307b9e7c989"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bb28ab5300cf974c7eb68ea21125c493e74b35b1129e629533468b2064ae0a2"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-win32.whl", hash = "sha256:1b1f74997b6d94d66375479fa55f70b1c18e4d865d7afcd13f0785bfd40a9d3c"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:c56d2efdfaa1c642029f3a7a5bb76085c5531f7a530777be98232d2ce142553c"}, - {file = "rapidfuzz-3.9.0-cp310-cp310-win_arm64.whl", hash = "sha256:6a83128d505cac76ea560bb9afcb3f6986e14e50a6f467db9a31faef4bd9b347"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e2218d62ab63f3c5ad48eced898854d0c2c327a48f0fb02e2288d7e5332a22c8"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36bf35df2d6c7d5820da20a6720aee34f67c15cd2daf8cf92e8141995c640c25"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:905b01a9b633394ff6bb5ebb1c5fd660e0e180c03fcf9d90199cc6ed74b87cf7"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33cfabcb7fd994938a6a08e641613ce5fe46757832edc789c6a5602e7933d6fa"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1179dcd3d150a67b8a678cd9c84f3baff7413ff13c9e8fe85e52a16c97e24c9b"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47d97e28c42f1efb7781993b67c749223f198f6653ef177a0c8f2b1c516efcaf"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28da953eb2ef9ad527e536022da7afff6ace7126cdd6f3e21ac20f8762e76d2c"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:182b4e11de928fb4834e8f8b5ecd971b5b10a86fabe8636ab65d3a9b7e0e9ca7"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c74f2da334ce597f31670db574766ddeaee5d9430c2c00e28d0fbb7f76172036"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:014ac55b03f4074f903248ded181f3000f4cdbd134e6155cbf643f0eceb4f70f"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c4ef34b2ddbf448f1d644b4ec6475df8bbe5b9d0fee173ff2e87322a151663bd"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fc02157f521af15143fae88f92ef3ddcc4e0cff05c40153a9549dc0fbdb9adb3"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ff08081c49b18ba253a99e6a47f492e6ee8019e19bbb6ddc3ed360cd3ecb2f62"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-win32.whl", hash = "sha256:b9bf90b3d96925cbf8ef44e5ee3cf39ef0c422f12d40f7a497e91febec546650"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:d5d5684f54d82d9b0cf0b2701e55a630527a9c3dd5ddcf7a2e726a475ac238f2"}, - {file = "rapidfuzz-3.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:a2de844e0e971d7bd8aa41284627dbeacc90e750b90acfb016836553c7a63192"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f81fe99a69ac8ee3fd905e70c62f3af033901aeb60b69317d1d43d547b46e510"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:633b9d03fc04abc585c197104b1d0af04b1f1db1abc99f674d871224cd15557a"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab872cb57ae97c54ba7c71a9e3c9552beb57cb907c789b726895576d1ea9af6f"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdd8c15c3a14e409507fdf0c0434ec481d85c6cbeec8bdcd342a8cd1eda03825"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2444d8155d9846f206e2079bb355b85f365d9457480b0d71677a112d0a7f7128"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f83bd3d01f04061c3660742dc85143a89d49fd23eb31eccbf60ad56c4b955617"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ca799f882364e69d0872619afb19efa3652b7133c18352e4a3d86a324fb2bb1"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6993d361f28b9ef5f0fa4e79b8541c2f3507be7471b9f9cb403a255e123b31e1"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:170822a1b1719f02b58e3dce194c8ad7d4c5b39be38c0fdec603bd19c6f9cf81"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e86e39c1c1a0816ceda836e6f7bd3743b930cbc51a43a81bb433b552f203f25"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:731269812ea837e0b93d913648e404736407408e33a00b75741e8f27c590caa2"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8e5ff882d3a3d081157ceba7e0ebc7fac775f95b08cbb143accd4cece6043819"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2003071aa633477a01509890c895f9ef56cf3f2eaa72c7ec0b567f743c1abcba"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-win32.whl", hash = "sha256:13857f9070600ea1f940749f123b02d0b027afbaa45e72186df0f278915761d0"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:134b7098ac109834eeea81424b6822f33c4c52bf80b81508295611e7a21be12a"}, - {file = "rapidfuzz-3.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:2a96209f046fe328be30fc43f06e3d4b91f0d5b74e9dcd627dbfd65890fa4a5e"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:544b0bf9d17170720809918e9ccd0d482d4a3a6eca35630d8e1459f737f71755"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d536f8beb8dd82d6efb20fe9f82c2cfab9ffa0384b5d184327e393a4edde91d"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:30f7609da871510583f87484a10820b26555a473a90ab356cdda2f3b4456256c"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f4a2468432a1db491af6f547fad8f6d55fa03e57265c2f20e5eaceb68c7907e"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a7ec4676242c8a430509cff42ce98bca2fbe30188a63d0f60fdcbfd7e84970"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dcb523243e988c849cf81220164ec3bbed378a699e595a8914fffe80596dc49f"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4eea3bf72c4fe68e957526ffd6bcbb403a21baa6b3344aaae2d3252313df6199"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4514980a5d204c076dd5b756960f6b1b7598f030009456e6109d76c4c331d03c"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9a06a99f1335fe43464d7121bc6540de7cd9c9475ac2025babb373fe7f27846b"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6c1ed63345d1581c39d4446b1a8c8f550709656ce2a3c88c47850b258167f3c2"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:cd2e6e97daf17ebb3254285cf8dd86c60d56d6cf35c67f0f9a557ef26bd66290"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9bc0f7e6256a9c668482c41c8a3de5d0aa12e8ca346dcc427b97c7edb82cba48"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c09f4e87e82a164c9db769474bc61f8c8b677f2aeb0234b8abac73d2ecf9799"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-win32.whl", hash = "sha256:e65b8f7921bf60cbb207c132842a6b45eefef48c4c3b510eb16087d6c08c70af"}, - {file = "rapidfuzz-3.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9d6478957fb35c7844ad08f2442b62ba76c1857a56370781a707eefa4f4981e1"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:65d9250a4b0bf86320097306084bc3ca479c8f5491927c170d018787793ebe95"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:47b7c0840afa724db3b1a070bc6ed5beab73b4e659b1d395023617fc51bf68a2"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3a16c48c6df8fb633efbbdea744361025d01d79bca988f884a620e63e782fe5b"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48105991ff6e4a51c7f754df500baa070270ed3d41784ee0d097549bc9fcb16d"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a7f273906b3c7cc6d63a76e088200805947aa0bc1ada42c6a0e582e19c390d7"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c396562d304e974b4b0d5cd3afc4f92c113ea46a36e6bc62e45333d6aa8837e"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68da1b70458fea5290ec9a169fcffe0c17ff7e5bb3c3257e63d7021a50601a8e"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c5b8f9a7b177af6ce7c6ad5b95588b4b73e37917711aafa33b2e79ee80fe709"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3c42a238bf9dd48f4ccec4c6934ac718225b00bb3a438a008c219e7ccb3894c7"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a365886c42177b2beab475a50ba311b59b04f233ceaebc4c341f6f91a86a78e2"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ce897b5dafb7fb7587a95fe4d449c1ea0b6d9ac4462fbafefdbbeef6eee4cf6a"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:413ac49bae291d7e226a5c9be65c71b2630b3346bce39268d02cb3290232e4b7"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8982fc3bd49d55a91569fc8a3feba0de4cef0b391ff9091be546e9df075b81"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-win32.whl", hash = "sha256:3904d0084ab51f82e9f353031554965524f535522a48ec75c30b223eb5a0a488"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:3733aede16ea112728ffeafeb29ccc62e095ed8ec816822fa2a82e92e2c08696"}, - {file = "rapidfuzz-3.9.0-cp39-cp39-win_arm64.whl", hash = "sha256:fc4e26f592b51f97acf0a3f8dfed95e4d830c6a8fbf359361035df836381ab81"}, - {file = "rapidfuzz-3.9.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e33362e98c7899b5f60dcb06ada00acd8673ce0d59aefe9a542701251fd00423"}, - {file = "rapidfuzz-3.9.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb67cf43ad83cb886cbbbff4df7dcaad7aedf94d64fca31aea0da7d26684283c"}, - {file = "rapidfuzz-3.9.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e2e106cc66453bb80d2ad9c0044f8287415676df5c8036d737d05d4b9cdbf8e"}, - {file = "rapidfuzz-3.9.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1256915f7e7a5cf2c151c9ac44834b37f9bd1c97e8dec6f936884f01b9dfc7d"}, - {file = "rapidfuzz-3.9.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ae643220584518cbff8bf2974a0494d3e250763af816b73326a512c86ae782ce"}, - {file = "rapidfuzz-3.9.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:491274080742110427f38a6085bb12dffcaff1eef12dccf9e8758398c7e3957e"}, - {file = "rapidfuzz-3.9.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bc5559b9b94326922c096b30ae2d8e5b40b2e9c2c100c2cc396ad91bcb84d30"}, - {file = "rapidfuzz-3.9.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:849160dc0f128acb343af514ca827278005c1d00148d025e4035e034fc2d8c7f"}, - {file = "rapidfuzz-3.9.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:623883fb78e692d54ed7c43b09beec52c6685f10a45a7518128e25746667403b"}, - {file = "rapidfuzz-3.9.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d20ab9abc7e19767f1951772a6ab14cb4eddd886493c2da5ee12014596ad253f"}, - {file = "rapidfuzz-3.9.0.tar.gz", hash = "sha256:b182f0fb61f6ac435e416eb7ab330d62efdbf9b63cf0c7fa12d1f57c2eaaf6f3"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f897a9bff517d5c6af6a90131796b4298b547b9a9a4df3cf285006be33aae5b"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83c570ce23b447625929c0e7c4f2eab6d90f5a576db2b26a5aa0594a53d560ea"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4c6b6455ee8404a663e15477a8bfe98b1afb329ff224bcf6d15f623a3761b95"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa44aef769e5834fef4fde091fd646cc1c52a2813b3aa241ae54b3028960abaa"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25ea055ae40fb60f503f02b44b3ac35a39a9108be33f89e05b81bc4e3c849ec8"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb592bad9d58b47c6681f0c180767d2c98775a35f7267131d33723139c3d6c2e"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb023adfefa62410fff877f7cc70cd4758cbfbad963e87d146cf71b022dce197"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c883d2d2e31c759af1f3fdeb67ec151cf94e307f745b3d02ab3a2ef6595485f2"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:8e11405d81e8baea4999a5757a982009566cff8f6a121d5ccf042aab81ae0230"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:02ed579f35ddd3552c7f74bc0c10800b432d9b09a4cebb19fd7a10b3b4759cc0"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9f8615a2a67a1f80b3aa7a3d7fbe6a2ed062a54c98988e3f9b664b49a3bc115e"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:64058f4a3698c6c8464df47a3b7da303db2477b2447142da3e67fc091f4c366a"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-win32.whl", hash = "sha256:2ef42c43c94139c890aeec40bc442c4bf8d48e15b456a88ce0f4cc5cfcad1896"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:c99d001c45c31c2cd2f7361bc2036d3062b21db5f43beea8bc5109d43fe9f283"}, + {file = "rapidfuzz-3.9.1-cp310-cp310-win_arm64.whl", hash = "sha256:da3f495cf4f7a443b34a6d3c6805265595fcd13641b3253a8e2034289d828dd9"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8869dcf072227a40a6f9e87b3fc4eb020055a08ad12b63d751c354e3a973ccb"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f66f9d5f14141b4b017e76118ec4bda29266f6b281989026e3a9ba1a2aaf032"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07decc6b058f935d2219423a50aac426027928cc734809f793bc250de4a3756e"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c125095d1828fa10ac79077594dd2d8829167d9e184e20baa97620fc52ebdcc9"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76a55bcc3abc9f8e38a1218cb5a09719126cfc4cba23ebd8caa27dfdc69cedd8"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50c2f7ad132dfeb6247c90b41431662af939a820f761cf930708d55912377ed8"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177bddf50577db59bcb00b6f7a5c2b70f2ec5a2aba40c8add7a6f7fd8609224e"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dec2792f864be731c8339cad99001caa6540aa909e6fd8bc688bb0419c501f44"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6437cba4b9460d5ee0bafd796e13ef9307091b81685bbe745b0f1619fb887ca"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:874317057a58a9c6ddf59fe1491e478217daa9fdb043a00358a15de4f62f9a2d"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5d8eb7fe39e81dc1530a3ec81a35e69770839c76607c461eb9d0902427fab3e1"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08f85d6674d804a493c3e9ec10a807f9bd8f482781487eda064913b537f99d7f"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-win32.whl", hash = "sha256:eadf8c4b24b63aef8810ed585c24ac1fc022ee771211772a6e9f78c63aa949ff"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:0e931539edeb9158ef83537cd571051f8a9608737642c20b088a37bd5d76c5c9"}, + {file = "rapidfuzz-3.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:bcc0ffcaeb1e499e708f32ec30177ed690b3f25455c91ad8c2240986c69f9ebe"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8781e45c56f7f3a64940f4d594a4ffd69360147925a706569b2b0c57347b2225"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0edc950c6a04c19db10670cd04e33403b3eb0f175deb620f9668595d378b1005"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ceb0d7bdec910d93793d32633ba0cb644356cf6778f9d91b727da0075beaec1"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a231e8f3bae82f10e7188965b37c91d8bfb80136595c860c8a08eb0dd07764d"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bfa8c0a8ce09b4bcd36322f8f375750dca160fbdbeb2e763a695cef3ae9133e"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e872627d5359c840f3e431b0beb263518048917c3e076f624870552d84e7dc6"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f006c3af82c478df09a790fb4846b5acd00a187d75715674d71f5dc0ac982ce"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:32718fa69306df969bf4fca1719f8900b83df315a2a8153942d5b8906f4fd1d6"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a89b219c1a7933a0673b2dbb1ffe701057d82e5cb843552be4f55b61b557031e"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b26cc9459e096959fab3a4a8a17b96a6c7c961f9db5c37c1c3c7a06789316cf7"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2f9044a4470343087cde10beaa36266519d5da110a9a4597b43e6aa35fa928d3"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a3a48fc6dc274b803a366a4baec99e212792ae1b1e73d42235b2042cd3ade7c1"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-win32.whl", hash = "sha256:b71e7f99ed048a338e4a1ac34f56b3b3933a3ba2dfbb04450c786a8ddd97f4db"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:e58489934d0147f1edda693cb983bea40f2b45ae6756fd47c1005b538f817a2f"}, + {file = "rapidfuzz-3.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:829fbad93266fffa0f9d722a94cbb1b95b53e3c04be4e872193496a0cfbd66f0"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bca2b93c75f87cd85832cdd5bb06b4b5642e2a05c8e3550841ddf5d564ce4abb"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d3397630f22e6c60dda8be3e9dbcf6a341695d487df8a6c92f4a2f7ebcdaecf7"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7bba8d4a8fb8e7559a9e83dfc5385dc6fe89efd73e32d253667242faf1883c"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e70605350cb6ec5091e06de62d3dcb058f694b059b4e1a9d85bfbf892f70030"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:678fcaa5117ddb6263160a7c5f33cc9ea3df335465f5d53715707fad103e1d09"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08dcd347d408912b6da778a73a0d7a2adad7fe238a44263e5e3789f2a8d84669"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7149afaf0294882b6b15bb6fa9fc38ff1d761e50117460ee3561181c1c4e2230"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:0c568b89a5016e76f0b3f85e9379036da99c5e7ec26b33935453d353a1938b74"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:a079164675d24eb715230bf9dd252683ae3c9c0c0a236f0b8098630268b899e9"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f6f4e8235d0acf1972f5eb4091c4a0473e5670a754f166c0c718ce21e945f879"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:ea62c82eb2c65bd49651e95f4f46874483ae4da1c3b57997e58f1b4fb2de6c05"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3ab6ad7e70469aed24e24378b19a9e47fc757c847399b22c612a0fccacc795cb"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-win32.whl", hash = "sha256:bf5184b17e26a82b00c7ee05d9ec5d826113df55830bbc447bf6d6e7469c70fb"}, + {file = "rapidfuzz-3.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:493354f50b9855271ac846b213e394e08446e70cef5cc033e5302a2220f3ae7b"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c8b0e6640421e55d69e186ce7fb9e6c723cfd3b6f91beaeb28705c2a46c8a194"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bc68fb8f2a8b5b3a4526b7a65e7d5c7f821882f56d9dcbcce4c6859a9e5bdcd7"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77767b119ac05662d216a8cc4092ac28dbc015d9caabebdbefe371b0dd82a38e"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dbcc4add07bd60ea73b94392fed28f83dba0fe796097da47627fd539bd6daca"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c0899de4fc1a7a36f14be556a0dacf40ea5c0fe22c6b45b2ea2674e1ac47e269"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9f075366cee63a6b06bd7f9285eb9f1785382a6493afcb7054202e20508bf94"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:349e9c15092d20a1f6ff1795e068f39a9ee5e84c54b3addbc66d0ac469c4ef43"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a645f362dafc103dbe7f43a2ad34f76284773cd7d1b00514d1c591848a1c817f"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:03a0a4bc8d4bd3e6f882b4c2ac183825a9b6dabe7e5a97bb6a1075e4635c944d"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:fbce66cb2e331b0888c79b594eab76e2c609c2637050085daadff5325d471dc2"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e5d7b3154f6df9e05c2016de5e95f8cba4fe636a4e5520ebcd89bc6c54b8e4ed"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:aaba665f92c011c6f284e933ab02b5dc129a6d3f48fce913ec4a214bd530135e"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-win32.whl", hash = "sha256:61b343c581f4926260248069d8fdbbbf293c19c12ef440ad5ced15bcff277a84"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:ce3335324198e1388a1c4e50d40f45107367010afe9fa09fd46278160f0ab591"}, + {file = "rapidfuzz-3.9.1-cp39-cp39-win_arm64.whl", hash = "sha256:998977df2ae01ff8b7bc3b29a860b4a863005e0533e323df3fd555a31ef33f0e"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:dc4b5de5d6f9347d836d849b56bca630169353cbe5c10fa7fe93bb1677b49770"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9f74d93148081049ccc86f276d54cd7c8c0692250245660b4fcd904ed1db1e01"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f9dfdcd75e16e5874efee233b28aec1322623b0f1f20641452d06ea2d8ba5ef"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97ab8f153984a5d827ebb5a5b80ee59563efcf2fa3e569dcd46ea7e7c9845e93"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8a6d5a8edc452920efdf1b499a2a47bb8a28440f7ab3fe28bb7d6636ccf71c3"}, + {file = "rapidfuzz-3.9.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:06879b598e798a4d33a283c2b4fa0d555d7706b6531e3321b161d62e986f7f57"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0f906ab6220778404498e0ce255c4cc89f98ea5e656e54cc59c5813c877eb86b"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d3da444890c9559fd15717d97f8373b1cd14007f68c9b037aa93ef7ca969b559"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc4503841cd3cbe22b5ac44f15bc834ec97d811a3c3943f73f5643266c8674e1"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5713c56b30ed75ada3a8f008cf8e8e6323386ce48fac2bf2d07285fe6c91f5a4"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb3f1af099cd1d98001691fbdadd422f088f21eadcacf5698b393b7569e24dc4"}, + {file = "rapidfuzz-3.9.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:53477e1e6d85d603c9a319cfd00ab9f0a57b6d68bcdb268d6b15a79e64d693d0"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a47550eabf235e5d50e7d448c18f77f6e8082aa3571e9df511c8388525ea9372"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c6b11a38b61cc2462a113b123f5e932cda0e525f816d6fe4b68516f97d7f9d49"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:414644a2fc8a3e5fafda95b430214ed892faa4d0a07401d33892bc9ca5c84974"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1539e7439b68013c5d2ab7ed9d3d221480a15595207764145ae177077d28016d"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e18f0e9351f7e5d5387774ff4d5cabd824341e16b866eb1c8d3f557111b447ef"}, + {file = "rapidfuzz-3.9.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d2ff268a8bf57a76512804d5ca2097afaf98e64e8947d514cde7e2e8446aa5f7"}, + {file = "rapidfuzz-3.9.1.tar.gz", hash = "sha256:a42eb645241f39a59c45a7fc15e3faf61886bff3a4a22263fd0f7cfb90e91b7f"}, ] [package.extras] @@ -3358,101 +3368,101 @@ full = ["numpy"] [[package]] name = "regex" -version = "2024.5.10" +version = "2024.5.15" description = "Alternative regular expression module, to replace re." optional = true python-versions = ">=3.8" files = [ - {file = "regex-2024.5.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eda3dd46df535da787ffb9036b5140f941ecb91701717df91c9daf64cabef953"}, - {file = "regex-2024.5.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d5bd666466c8f00a06886ce1397ba8b12371c1f1c6d1bef11013e9e0a1464a8"}, - {file = "regex-2024.5.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:32e5f3b8e32918bfbdd12eca62e49ab3031125c454b507127ad6ecbd86e62fca"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:534efd2653ebc4f26fc0e47234e53bf0cb4715bb61f98c64d2774a278b58c846"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:193b7c6834a06f722f0ce1ba685efe80881de7c3de31415513862f601097648c"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:160ba087232c5c6e2a1e7ad08bd3a3f49b58c815be0504d8c8aacfb064491cd8"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:951be1eae7b47660412dc4938777a975ebc41936d64e28081bf2e584b47ec246"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8a0f0ab5453e409586b11ebe91c672040bc804ca98d03a656825f7890cbdf88"}, - {file = "regex-2024.5.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9e6d4d6ae1827b2f8c7200aaf7501c37cf3f3896c86a6aaf2566448397c823dd"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:161a206c8f3511e2f5fafc9142a2cc25d7fe9a1ec5ad9b4ad2496a7c33e1c5d2"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:44b3267cea873684af022822195298501568ed44d542f9a2d9bebc0212e99069"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:560278c9975694e1f0bc50da187abf2cdc1e4890739ea33df2bc4a85eeef143e"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:70364a097437dd0a90b31cd77f09f7387ad9ac60ef57590971f43b7fca3082a5"}, - {file = "regex-2024.5.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42be5de7cc8c1edac55db92d82b68dc8e683b204d6f5414c5a51997a323d7081"}, - {file = "regex-2024.5.10-cp310-cp310-win32.whl", hash = "sha256:9a8625849387b9d558d528e263ecc9c0fbde86cfa5c2f0eef43fff480ae24d71"}, - {file = "regex-2024.5.10-cp310-cp310-win_amd64.whl", hash = "sha256:903350bf44d7e4116b4d5898b30b15755d61dcd3161e3413a49c7db76f0bee5a"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bf9596cba92ce7b1fd32c7b07c6e3212c7eed0edc271757e48bfcd2b54646452"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:45cc13d398b6359a7708986386f72bd156ae781c3e83a68a6d4cee5af04b1ce9"}, - {file = "regex-2024.5.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad45f3bccfcb00868f2871dce02a755529838d2b86163ab8a246115e80cfb7d6"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33d19f0cde6838c81acffff25c7708e4adc7dd02896c9ec25c3939b1500a1778"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a9f89d7db5ef6bdf53e5cc8e6199a493d0f1374b3171796b464a74ebe8e508a"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c6c71cf92b09e5faa72ea2c68aa1f61c9ce11cb66fdc5069d712f4392ddfd00"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7467ad8b0eac0b28e52679e972b9b234b3de0ea5cee12eb50091d2b68145fe36"}, - {file = "regex-2024.5.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc0db93ad039fc2fe32ccd3dd0e0e70c4f3d6e37ae83f0a487e1aba939bd2fbd"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fa9335674d7c819674467c7b46154196c51efbaf5f5715187fd366814ba3fa39"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7dda3091838206969c2b286f9832dff41e2da545b99d1cfaea9ebd8584d02708"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:504b5116e2bd1821efd815941edff7535e93372a098e156bb9dffde30264e798"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:91b53dea84415e8115506cc62e441a2b54537359c63d856d73cb1abe05af4c9a"}, - {file = "regex-2024.5.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a3903128f9e17a500618e80c68165c78c741ebb17dd1a0b44575f92c3c68b02"}, - {file = "regex-2024.5.10-cp311-cp311-win32.whl", hash = "sha256:236cace6c1903effd647ed46ce6dd5d76d54985fc36dafc5256032886736c85d"}, - {file = "regex-2024.5.10-cp311-cp311-win_amd64.whl", hash = "sha256:12446827f43c7881decf2c126762e11425de5eb93b3b0d8b581344c16db7047a"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:14905ed75c7a6edf423eb46c213ed3f4507c38115f1ed3c00f4ec9eafba50e58"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4fad420b14ae1970a1f322e8ae84a1d9d89375eb71e1b504060ab2d1bfe68f3c"}, - {file = "regex-2024.5.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c46a76a599fcbf95f98755275c5527304cc4f1bb69919434c1e15544d7052910"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0faecb6d5779753a6066a3c7a0471a8d29fe25d9981ca9e552d6d1b8f8b6a594"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aab65121229c2ecdf4a31b793d99a6a0501225bd39b616e653c87b219ed34a49"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50e7e96a527488334379e05755b210b7da4a60fc5d6481938c1fa053e0c92184"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba034c8db4b264ef1601eb33cd23d87c5013b8fb48b8161debe2e5d3bd9156b0"}, - {file = "regex-2024.5.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:031219782d97550c2098d9a68ce9e9eaefe67d2d81d8ff84c8354f9c009e720c"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:62b5f7910b639f3c1d122d408421317c351e213ca39c964ad4121f27916631c6"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cd832bd9b6120d6074f39bdfbb3c80e416848b07ac72910f1c7f03131a6debc3"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:e91b1976358e17197157b405cab408a5f4e33310cda211c49fc6da7cffd0b2f0"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:571452362d552de508c37191b6abbbb660028b8b418e2d68c20779e0bc8eaaa8"}, - {file = "regex-2024.5.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5253dcb0bfda7214523de58b002eb0090cb530d7c55993ce5f6d17faf953ece7"}, - {file = "regex-2024.5.10-cp312-cp312-win32.whl", hash = "sha256:2f30a5ab8902f93930dc6f627c4dd5da2703333287081c85cace0fc6e21c25af"}, - {file = "regex-2024.5.10-cp312-cp312-win_amd64.whl", hash = "sha256:3799e36d60a35162bb35b2246d8bb012192b7437dff807ef79c14e7352706306"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bbdc5db2c98ac2bf1971ffa1410c87ca7a15800415f788971e8ba8520fc0fda9"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6ccdeef4584450b6f0bddd5135354908dacad95425fcb629fe36d13e48b60f32"}, - {file = "regex-2024.5.10-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:29d839829209f3c53f004e1de8c3113efce6d98029f044fa5cfee666253ee7e6"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0709ba544cf50bd5cb843df4b8bb6701bae2b70a8e88da9add8386cbca5c1385"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:972b49f2fe1047b9249c958ec4fa1bdd2cf8ce305dc19d27546d5a38e57732d8"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cdbb1998da94607d5eec02566b9586f0e70d6438abf1b690261aac0edda7ab6"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7c8ee4861d9ef5b1120abb75846828c811f932d63311596ad25fa168053e00"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d35d4cc9270944e95f9c88af757b0c9fc43f396917e143a5756608462c5223b"}, - {file = "regex-2024.5.10-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8722f72068b3e1156a4b2e1afde6810f1fc67155a9fa30a4b9d5b4bc46f18fb0"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:696639a73ca78a380acfaa0a1f6dd8220616a99074c05bba9ba8bb916914b224"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea057306ab469130167014b662643cfaed84651c792948891d003cf0039223a5"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b43b78f9386d3d932a6ce5af4b45f393d2e93693ee18dc4800d30a8909df700e"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c43395a3b7cc9862801a65c6994678484f186ce13c929abab44fb8a9e473a55a"}, - {file = "regex-2024.5.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0bc94873ba11e34837bffd7e5006703abeffc4514e2f482022f46ce05bd25e67"}, - {file = "regex-2024.5.10-cp38-cp38-win32.whl", hash = "sha256:1118ba9def608250250f4b3e3f48c62f4562ba16ca58ede491b6e7554bfa09ff"}, - {file = "regex-2024.5.10-cp38-cp38-win_amd64.whl", hash = "sha256:458d68d34fb74b906709735c927c029e62f7d06437a98af1b5b6258025223210"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:15e593386ec6331e0ab4ac0795b7593f02ab2f4b30a698beb89fbdc34f92386a"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ca23b41355ba95929e9505ee04e55495726aa2282003ed9b012d86f857d3e49b"}, - {file = "regex-2024.5.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c8982ee19ccecabbaeac1ba687bfef085a6352a8c64f821ce2f43e6d76a9298"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7117cb7d6ac7f2e985f3d18aa8a1728864097da1a677ffa69e970ca215baebf1"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b66421f8878a0c82fc0c272a43e2121c8d4c67cb37429b764f0d5ad70b82993b"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:224a9269f133564109ce668213ef3cb32bc72ccf040b0b51c72a50e569e9dc9e"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab98016541543692a37905871a5ffca59b16e08aacc3d7d10a27297b443f572d"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51d27844763c273a122e08a3e86e7aefa54ee09fb672d96a645ece0454d8425e"}, - {file = "regex-2024.5.10-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:853cc36e756ff673bf984e9044ccc8fad60b95a748915dddeab9488aea974c73"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e7eaf9df15423d07b6050fb91f86c66307171b95ea53e2d87a7993b6d02c7f7"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:169fd0acd7a259f58f417e492e93d0e15fc87592cd1e971c8c533ad5703b5830"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:334b79ce9c08f26b4659a53f42892793948a613c46f1b583e985fd5a6bf1c149"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f03b1dbd4d9596dd84955bb40f7d885204d6aac0d56a919bb1e0ff2fb7e1735a"}, - {file = "regex-2024.5.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfa6d61a76c77610ba9274c1a90a453062bdf6887858afbe214d18ad41cf6bde"}, - {file = "regex-2024.5.10-cp39-cp39-win32.whl", hash = "sha256:249fbcee0a277c32a3ce36d8e36d50c27c968fdf969e0fbe342658d4e010fbc8"}, - {file = "regex-2024.5.10-cp39-cp39-win_amd64.whl", hash = "sha256:0ce56a923f4c01d7568811bfdffe156268c0a7aae8a94c902b92fe34c4bde785"}, - {file = "regex-2024.5.10.tar.gz", hash = "sha256:304e7e2418146ae4d0ef0e9ffa28f881f7874b45b4994cc2279b21b6e7ae50c8"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a81e3cfbae20378d75185171587cbf756015ccb14840702944f014e0d93ea09f"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b59138b219ffa8979013be7bc85bb60c6f7b7575df3d56dc1e403a438c7a3f6"}, + {file = "regex-2024.5.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0bd000c6e266927cb7a1bc39d55be95c4b4f65c5be53e659537537e019232b1"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eaa7ddaf517aa095fa8da0b5015c44d03da83f5bd49c87961e3c997daed0de7"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba68168daedb2c0bab7fd7e00ced5ba90aebf91024dea3c88ad5063c2a562cca"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e8d717bca3a6e2064fc3a08df5cbe366369f4b052dcd21b7416e6d71620dca1"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1337b7dbef9b2f71121cdbf1e97e40de33ff114801263b275aafd75303bd62b5"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9ebd0a36102fcad2f03696e8af4ae682793a5d30b46c647eaf280d6cfb32796"}, + {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9efa1a32ad3a3ea112224897cdaeb6aa00381627f567179c0314f7b65d354c62"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1595f2d10dff3d805e054ebdc41c124753631b6a471b976963c7b28543cf13b0"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b802512f3e1f480f41ab5f2cfc0e2f761f08a1f41092d6718868082fc0d27143"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a0981022dccabca811e8171f913de05720590c915b033b7e601f35ce4ea7019f"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:19068a6a79cf99a19ccefa44610491e9ca02c2be3305c7760d3831d38a467a6f"}, + {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1b5269484f6126eee5e687785e83c6b60aad7663dafe842b34691157e5083e53"}, + {file = "regex-2024.5.15-cp310-cp310-win32.whl", hash = "sha256:ada150c5adfa8fbcbf321c30c751dc67d2f12f15bd183ffe4ec7cde351d945b3"}, + {file = "regex-2024.5.15-cp310-cp310-win_amd64.whl", hash = "sha256:ac394ff680fc46b97487941f5e6ae49a9f30ea41c6c6804832063f14b2a5a145"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f5b1dff3ad008dccf18e652283f5e5339d70bf8ba7c98bf848ac33db10f7bc7a"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c6a2b494a76983df8e3d3feea9b9ffdd558b247e60b92f877f93a1ff43d26656"}, + {file = "regex-2024.5.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a32b96f15c8ab2e7d27655969a23895eb799de3665fa94349f3b2fbfd547236f"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10002e86e6068d9e1c91eae8295ef690f02f913c57db120b58fdd35a6bb1af35"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec54d5afa89c19c6dd8541a133be51ee1017a38b412b1321ccb8d6ddbeb4cf7d"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10e4ce0dca9ae7a66e6089bb29355d4432caed736acae36fef0fdd7879f0b0cb"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e507ff1e74373c4d3038195fdd2af30d297b4f0950eeda6f515ae3d84a1770f"}, + {file = "regex-2024.5.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f059a4d795e646e1c37665b9d06062c62d0e8cc3c511fe01315973a6542e40"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0721931ad5fe0dda45d07f9820b90b2148ccdd8e45bb9e9b42a146cb4f695649"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:833616ddc75ad595dee848ad984d067f2f31be645d603e4d158bba656bbf516c"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:287eb7f54fc81546346207c533ad3c2c51a8d61075127d7f6d79aaf96cdee890"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:19dfb1c504781a136a80ecd1fff9f16dddf5bb43cec6871778c8a907a085bb3d"}, + {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:119af6e56dce35e8dfb5222573b50c89e5508d94d55713c75126b753f834de68"}, + {file = "regex-2024.5.15-cp311-cp311-win32.whl", hash = "sha256:1c1c174d6ec38d6c8a7504087358ce9213d4332f6293a94fbf5249992ba54efa"}, + {file = "regex-2024.5.15-cp311-cp311-win_amd64.whl", hash = "sha256:9e717956dcfd656f5055cc70996ee2cc82ac5149517fc8e1b60261b907740201"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:632b01153e5248c134007209b5c6348a544ce96c46005d8456de1d552455b014"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e64198f6b856d48192bf921421fdd8ad8eb35e179086e99e99f711957ffedd6e"}, + {file = "regex-2024.5.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68811ab14087b2f6e0fc0c2bae9ad689ea3584cad6917fc57be6a48bbd012c49"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ec0c2fea1e886a19c3bee0cd19d862b3aa75dcdfb42ebe8ed30708df64687a"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0c0c0003c10f54a591d220997dd27d953cd9ccc1a7294b40a4be5312be8797b"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2431b9e263af1953c55abbd3e2efca67ca80a3de8a0437cb58e2421f8184717a"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a605586358893b483976cffc1723fb0f83e526e8f14c6e6614e75919d9862cf"}, + {file = "regex-2024.5.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391d7f7f1e409d192dba8bcd42d3e4cf9e598f3979cdaed6ab11288da88cb9f2"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9ff11639a8d98969c863d4617595eb5425fd12f7c5ef6621a4b74b71ed8726d5"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4eee78a04e6c67e8391edd4dad3279828dd66ac4b79570ec998e2155d2e59fd5"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8fe45aa3f4aa57faabbc9cb46a93363edd6197cbc43523daea044e9ff2fea83e"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d0a3d8d6acf0c78a1fff0e210d224b821081330b8524e3e2bc5a68ef6ab5803d"}, + {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c486b4106066d502495b3025a0a7251bf37ea9540433940a23419461ab9f2a80"}, + {file = "regex-2024.5.15-cp312-cp312-win32.whl", hash = "sha256:c49e15eac7c149f3670b3e27f1f28a2c1ddeccd3a2812cba953e01be2ab9b5fe"}, + {file = "regex-2024.5.15-cp312-cp312-win_amd64.whl", hash = "sha256:673b5a6da4557b975c6c90198588181029c60793835ce02f497ea817ff647cb2"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:87e2a9c29e672fc65523fb47a90d429b70ef72b901b4e4b1bd42387caf0d6835"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c3bea0ba8b73b71b37ac833a7f3fd53825924165da6a924aec78c13032f20850"}, + {file = "regex-2024.5.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfc4f82cabe54f1e7f206fd3d30fda143f84a63fe7d64a81558d6e5f2e5aaba9"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5bb9425fe881d578aeca0b2b4b3d314ec88738706f66f219c194d67179337cb"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64c65783e96e563103d641760664125e91bd85d8e49566ee560ded4da0d3e704"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf2430df4148b08fb4324b848672514b1385ae3807651f3567871f130a728cc3"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5397de3219a8b08ae9540c48f602996aa6b0b65d5a61683e233af8605c42b0f2"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:455705d34b4154a80ead722f4f185b04c4237e8e8e33f265cd0798d0e44825fa"}, + {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2b6f1b3bb6f640c1a92be3bbfbcb18657b125b99ecf141fb3310b5282c7d4ed"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3ad070b823ca5890cab606c940522d05d3d22395d432f4aaaf9d5b1653e47ced"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5b5467acbfc153847d5adb21e21e29847bcb5870e65c94c9206d20eb4e99a384"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e6662686aeb633ad65be2a42b4cb00178b3fbf7b91878f9446075c404ada552f"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:2b4c884767504c0e2401babe8b5b7aea9148680d2e157fa28f01529d1f7fcf67"}, + {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3cd7874d57f13bf70078f1ff02b8b0aa48d5b9ed25fc48547516c6aba36f5741"}, + {file = "regex-2024.5.15-cp38-cp38-win32.whl", hash = "sha256:e4682f5ba31f475d58884045c1a97a860a007d44938c4c0895f41d64481edbc9"}, + {file = "regex-2024.5.15-cp38-cp38-win_amd64.whl", hash = "sha256:d99ceffa25ac45d150e30bd9ed14ec6039f2aad0ffa6bb87a5936f5782fc1569"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13cdaf31bed30a1e1c2453ef6015aa0983e1366fad2667657dbcac7b02f67133"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cac27dcaa821ca271855a32188aa61d12decb6fe45ffe3e722401fe61e323cd1"}, + {file = "regex-2024.5.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7dbe2467273b875ea2de38ded4eba86cbcbc9a1a6d0aa11dcf7bd2e67859c435"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f18a9a3513a99c4bef0e3efd4c4a5b11228b48aa80743be822b71e132ae4f5"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d347a741ea871c2e278fde6c48f85136c96b8659b632fb57a7d1ce1872547600"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1878b8301ed011704aea4c806a3cadbd76f84dece1ec09cc9e4dc934cfa5d4da"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4babf07ad476aaf7830d77000874d7611704a7fcf68c9c2ad151f5d94ae4bfc4"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35cb514e137cb3488bce23352af3e12fb0dbedd1ee6e60da053c69fb1b29cc6c"}, + {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdd09d47c0b2efee9378679f8510ee6955d329424c659ab3c5e3a6edea696294"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:72d7a99cd6b8f958e85fc6ca5b37c4303294954eac1376535b03c2a43eb72629"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:a094801d379ab20c2135529948cb84d417a2169b9bdceda2a36f5f10977ebc16"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c18345010870e58238790a6779a1219b4d97bd2e77e1140e8ee5d14df071aa"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:16093f563098448ff6b1fa68170e4acbef94e6b6a4e25e10eae8598bb1694b5d"}, + {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e38a7d4e8f633a33b4c7350fbd8bad3b70bf81439ac67ac38916c4a86b465456"}, + {file = "regex-2024.5.15-cp39-cp39-win32.whl", hash = "sha256:71a455a3c584a88f654b64feccc1e25876066c4f5ef26cd6dd711308aa538694"}, + {file = "regex-2024.5.15-cp39-cp39-win_amd64.whl", hash = "sha256:cab12877a9bdafde5500206d1020a584355a97884dfd388af3699e9137bf7388"}, + {file = "regex-2024.5.15.tar.gz", hash = "sha256:d3ee02d9e5f482cc8309134a91eeaacbdd2261ba111b0fef3748eeb4913e6a2c"}, ] [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -3591,45 +3601,48 @@ files = [ [[package]] name = "scikit-learn" -version = "1.4.2" +version = "1.5.0" description = "A set of python modules for machine learning and data mining" optional = true python-versions = ">=3.9" files = [ - {file = "scikit-learn-1.4.2.tar.gz", hash = "sha256:daa1c471d95bad080c6e44b4946c9390a4842adc3082572c20e4f8884e39e959"}, - {file = "scikit_learn-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8539a41b3d6d1af82eb629f9c57f37428ff1481c1e34dddb3b9d7af8ede67ac5"}, - {file = "scikit_learn-1.4.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:68b8404841f944a4a1459b07198fa2edd41a82f189b44f3e1d55c104dbc2e40c"}, - {file = "scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81bf5d8bbe87643103334032dd82f7419bc8c8d02a763643a6b9a5c7288c5054"}, - {file = "scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36f0ea5d0f693cb247a073d21a4123bdf4172e470e6d163c12b74cbb1536cf38"}, - {file = "scikit_learn-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:87440e2e188c87db80ea4023440923dccbd56fbc2d557b18ced00fef79da0727"}, - {file = "scikit_learn-1.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:45dee87ac5309bb82e3ea633955030df9bbcb8d2cdb30383c6cd483691c546cc"}, - {file = "scikit_learn-1.4.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1d0b25d9c651fd050555aadd57431b53d4cf664e749069da77f3d52c5ad14b3b"}, - {file = "scikit_learn-1.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0203c368058ab92efc6168a1507d388d41469c873e96ec220ca8e74079bf62e"}, - {file = "scikit_learn-1.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44c62f2b124848a28fd695db5bc4da019287abf390bfce602ddc8aa1ec186aae"}, - {file = "scikit_learn-1.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:5cd7b524115499b18b63f0c96f4224eb885564937a0b3477531b2b63ce331904"}, - {file = "scikit_learn-1.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:90378e1747949f90c8f385898fff35d73193dfcaec3dd75d6b542f90c4e89755"}, - {file = "scikit_learn-1.4.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ff4effe5a1d4e8fed260a83a163f7dbf4f6087b54528d8880bab1d1377bd78be"}, - {file = "scikit_learn-1.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:671e2f0c3f2c15409dae4f282a3a619601fa824d2c820e5b608d9d775f91780c"}, - {file = "scikit_learn-1.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d36d0bc983336bbc1be22f9b686b50c964f593c8a9a913a792442af9bf4f5e68"}, - {file = "scikit_learn-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:d762070980c17ba3e9a4a1e043ba0518ce4c55152032f1af0ca6f39b376b5928"}, - {file = "scikit_learn-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9993d5e78a8148b1d0fdf5b15ed92452af5581734129998c26f481c46586d68"}, - {file = "scikit_learn-1.4.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:426d258fddac674fdf33f3cb2d54d26f49406e2599dbf9a32b4d1696091d4256"}, - {file = "scikit_learn-1.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5460a1a5b043ae5ae4596b3126a4ec33ccba1b51e7ca2c5d36dac2169f62ab1d"}, - {file = "scikit_learn-1.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49d64ef6cb8c093d883e5a36c4766548d974898d378e395ba41a806d0e824db8"}, - {file = "scikit_learn-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:c97a50b05c194be9146d61fe87dbf8eac62b203d9e87a3ccc6ae9aed2dfaf361"}, + {file = "scikit_learn-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12e40ac48555e6b551f0a0a5743cc94cc5a765c9513fe708e01f0aa001da2801"}, + {file = "scikit_learn-1.5.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f405c4dae288f5f6553b10c4ac9ea7754d5180ec11e296464adb5d6ac68b6ef5"}, + {file = "scikit_learn-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df8ccabbf583315f13160a4bb06037bde99ea7d8211a69787a6b7c5d4ebb6fc3"}, + {file = "scikit_learn-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c75ea812cd83b1385bbfa94ae971f0d80adb338a9523f6bbcb5e0b0381151d4"}, + {file = "scikit_learn-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:a90c5da84829a0b9b4bf00daf62754b2be741e66b5946911f5bdfaa869fcedd6"}, + {file = "scikit_learn-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2a65af2d8a6cce4e163a7951a4cfbfa7fceb2d5c013a4b593686c7f16445cf9d"}, + {file = "scikit_learn-1.5.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:4c0c56c3005f2ec1db3787aeaabefa96256580678cec783986836fc64f8ff622"}, + {file = "scikit_learn-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f77547165c00625551e5c250cefa3f03f2fc92c5e18668abd90bfc4be2e0bff"}, + {file = "scikit_learn-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:118a8d229a41158c9f90093e46b3737120a165181a1b58c03461447aa4657415"}, + {file = "scikit_learn-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:a03b09f9f7f09ffe8c5efffe2e9de1196c696d811be6798ad5eddf323c6f4d40"}, + {file = "scikit_learn-1.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:460806030c666addee1f074788b3978329a5bfdc9b7d63e7aad3f6d45c67a210"}, + {file = "scikit_learn-1.5.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:1b94d6440603752b27842eda97f6395f570941857456c606eb1d638efdb38184"}, + {file = "scikit_learn-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d82c2e573f0f2f2f0be897e7a31fcf4e73869247738ab8c3ce7245549af58ab8"}, + {file = "scikit_learn-1.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3a10e1d9e834e84d05e468ec501a356226338778769317ee0b84043c0d8fb06"}, + {file = "scikit_learn-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:855fc5fa8ed9e4f08291203af3d3e5fbdc4737bd617a371559aaa2088166046e"}, + {file = "scikit_learn-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:40fb7d4a9a2db07e6e0cae4dc7bdbb8fada17043bac24104d8165e10e4cff1a2"}, + {file = "scikit_learn-1.5.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:47132440050b1c5beb95f8ba0b2402bbd9057ce96ec0ba86f2f445dd4f34df67"}, + {file = "scikit_learn-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:174beb56e3e881c90424e21f576fa69c4ffcf5174632a79ab4461c4c960315ac"}, + {file = "scikit_learn-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261fe334ca48f09ed64b8fae13f9b46cc43ac5f580c4a605cbb0a517456c8f71"}, + {file = "scikit_learn-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:057b991ac64b3e75c9c04b5f9395eaf19a6179244c089afdebaad98264bff37c"}, + {file = "scikit_learn-1.5.0.tar.gz", hash = "sha256:789e3db01c750ed6d496fa2db7d50637857b451e57bcae863bff707c1247bef7"}, ] [package.dependencies] joblib = ">=1.2.0" numpy = ">=1.19.5" scipy = ">=1.6.0" -threadpoolctl = ">=2.0.0" +threadpoolctl = ">=3.1.0" [package.extras] -benchmark = ["matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "pandas (>=1.1.5)"] -docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.15.0)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] +benchmark = ["matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "pandas (>=1.1.5)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.15.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "polars (>=0.20.23)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.15.0)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] -tests = ["black (>=23.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.19.12)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.17.2)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.19.5)", "scipy (>=1.6.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==2.5.6)"] +tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.20.23)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.2.1)", "scikit-image (>=0.17.2)"] [[package]] name = "scipy" @@ -3675,19 +3688,18 @@ test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "po [[package]] name = "setuptools" -version = "69.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, - {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -4533,18 +4545,18 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.18.1" +version = "3.18.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.8" files = [ - {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, - {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, + {file = "zipp-3.18.2-py3-none-any.whl", hash = "sha256:dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e"}, + {file = "zipp-3.18.2.tar.gz", hash = "sha256:6278d9ddbcfb1f1089a88fde84481528b07b0e10474e09dcfe53dad4069fa059"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] file-based = ["avro", "fastavro", "markdown", "pdf2image", "pdfminer.six", "pyarrow", "pytesseract", "unstructured", "unstructured.pytesseract"] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py new file mode 100644 index 000000000000..efdb64c582eb --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py @@ -0,0 +1,21 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import pytest +from airbyte_cdk.sources.declarative.incremental import ResumableFullRefreshCursor +from airbyte_cdk.sources.types import StreamSlice + + +@pytest.mark.parametrize( + "stream_state, expected_slice", + [ + pytest.param({"updated_at": "2024-04-30"}, StreamSlice(cursor_slice={"updated_at": "2024-04-30"}, partition={}), id="test_set_incoming_stream_state"), + pytest.param({}, StreamSlice(cursor_slice={}, partition={}), id="test_empty_stream_state"), + ] +) +def test_stream_slices(stream_state, expected_slice): + cursor = ResumableFullRefreshCursor(parameters={}) + cursor.set_initial_state(stream_state=stream_state) + + actual_slices = [stream_slice for stream_slice in cursor.stream_slices()] + + assert actual_slices == [expected_slice] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 6306a854ec6e..fba43c12ea2c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -22,7 +22,7 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor, ResumableFullRefreshCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamModel from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel @@ -659,6 +659,137 @@ def test_stream_with_incremental_and_retriever_with_partition_router(): assert list_stream_slicer._cursor_field.string == "a_key" +def test_resumable_full_refresh_stream(): + content = """ +decoder: + type: JsonDecoder +extractor: + type: DpathExtractor + decoder: "#/decoder" +selector: + type: RecordSelector + record_filter: + type: RecordFilter + condition: "{{ record['id'] > stream_state['id'] }}" +metadata_paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + inject_into: request_parameter + field_name: page_size + page_token_option: + type: RequestPath + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + page_size: 10 +requester: + type: HttpRequester + url_base: "https://api.sendgrid.com/v3/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['apikey'] }}" + request_parameters: + unit: "day" +retriever: + paginator: + type: NoPagination +partial_stream: + type: DeclarativeStream + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_sendgrid/schemas/{{ parameters.name }}.json" +list_stream: + $ref: "#/partial_stream" + $parameters: + name: "lists" + extractor: + $ref: "#/extractor" + field_path: ["{{ parameters['name'] }}"] + name: "lists" + primary_key: "id" + retriever: + $ref: "#/retriever" + requester: + $ref: "#/requester" + path: "{{ next_page_token['next_page_url'] }}" + paginator: + $ref: "#/metadata_paginator" + record_selector: + $ref: "#/selector" + transformations: + - type: AddFields + fields: + - path: ["extra"] + value: "{{ response.to_add }}" +check: + type: CheckStream + stream_names: ["list_stream"] +spec: + type: Spec + documentation_url: https://airbyte.com/#yaml-from-manifest + connection_specification: + title: Test Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + airbyte_secret: true + title: API Key + description: Test API Key + order: 0 + advanced_auth: + auth_flow_type: "oauth2.0" + """ + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + resolved_manifest["type"] = "DeclarativeSource" + manifest = transformer.propagate_types_and_parameters("", resolved_manifest, {}) + + stream_manifest = manifest["list_stream"] + assert stream_manifest["type"] == "DeclarativeStream" + stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config) + + assert isinstance(stream, DeclarativeStream) + assert stream.primary_key == "id" + assert stream.name == "lists" + assert stream._stream_cursor_field.string == "" + + assert isinstance(stream.retriever, SimpleRetriever) + assert stream.retriever.primary_key == stream.primary_key + assert stream.retriever.name == stream.name + + assert isinstance(stream.retriever.record_selector, RecordSelector) + + assert isinstance(stream.retriever.stream_slicer, ResumableFullRefreshCursor) + assert isinstance(stream.retriever.cursor, ResumableFullRefreshCursor) + + assert isinstance(stream.retriever.paginator, DefaultPaginator) + assert isinstance(stream.retriever.paginator.decoder, JsonDecoder) + assert stream.retriever.paginator.page_size_option.field_name.eval(input_config) == "page_size" + assert stream.retriever.paginator.page_size_option.inject_into == RequestOptionType.request_parameter + assert isinstance(stream.retriever.paginator.page_token_option, RequestPath) + assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com/v3/" + assert stream.retriever.paginator.url_base.default == "https://api.sendgrid.com/v3/" + + assert isinstance(stream.retriever.paginator.pagination_strategy, CursorPaginationStrategy) + assert isinstance(stream.retriever.paginator.pagination_strategy.decoder, JsonDecoder) + assert stream.retriever.paginator.pagination_strategy._cursor_value.string == "{{ response._metadata.next }}" + assert stream.retriever.paginator.pagination_strategy._cursor_value.default == "{{ response._metadata.next }}" + assert stream.retriever.paginator.pagination_strategy.page_size == 10 + + checker = factory.create_component(model_type=CheckStreamModel, component_definition=manifest["check"], config=input_config) + + assert isinstance(checker, CheckStream) + streams_to_check = checker.stream_names + assert len(streams_to_check) == 1 + assert list(streams_to_check)[0] == "list_stream" + + def test_incremental_data_feed(): content = """ selector: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index dd7a9d2572ab..b9b0476c9625 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -90,3 +90,18 @@ def test_last_record_is_node_if_no_records(): response = requests.Response() next_page_token = strategy.next_page_token(response, 0, None) assert next_page_token is None + + +def test_reset_with_initial_token(): + strategy = CursorPaginationStrategy( + page_size=10, + cursor_value="{{ response.next_page }}", + config={}, + parameters={}, + ) + + assert strategy.initial_token is None + + strategy.reset("https://for-all-mankind.nasa.com/api/v1/astronauts") + + assert strategy.initial_token == "https://for-all-mankind.nasa.com/api/v1/astronauts" diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 326de7b7d197..d655dc9e6fa3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -56,3 +56,25 @@ def test_offset_increment_paginator_strategy_initial_token(inject_on_first_reque paginator_strategy = OffsetIncrement(page_size=20, parameters={}, config={}, inject_on_first_request=inject_on_first_request) assert paginator_strategy.initial_token == expected_initial_token + + +@pytest.mark.parametrize( + "reset_value, expected_initial_token, expected_error", + [ + pytest.param(25, 25, None, id="test_reset_with_offset_value"), + pytest.param(None, 0, None, id="test_reset_with_default"), + pytest.param("Nope", None, ValueError, id="test_reset_with_invalid_value"), + ], +) +def test_offset_increment_reset(reset_value, expected_initial_token, expected_error): + paginator_strategy = OffsetIncrement(page_size=20, parameters={}, config={}, inject_on_first_request=True) + + if expected_error: + with pytest.raises(expected_error): + paginator_strategy.reset(reset_value=reset_value) + else: + if reset_value is None: + paginator_strategy.reset() + else: + paginator_strategy.reset(reset_value=reset_value) + assert paginator_strategy.initial_token == expected_initial_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index eaa2b41ec07f..1ca14cc60481 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -68,3 +68,22 @@ def test_page_increment_paginator_strategy_initial_token( ) assert paginator_strategy.initial_token == expected_initial_token + + +@pytest.mark.parametrize( + "reset_value, expected_initial_token, expected_error", + [ + pytest.param(25, 25, None, id="test_reset_with_offset_value"), + pytest.param(None, 0, None, id="test_reset_with_default"), + pytest.param("Nope", None, ValueError, id="test_reset_with_invalid_value"), + ], +) +def test_offset_increment_reset(reset_value, expected_initial_token, expected_error): + paginator_strategy = PageIncrement(page_size=100, parameters={}, config={}, inject_on_first_request=True) + + if expected_error: + with pytest.raises(expected_error): + paginator_strategy.reset(reset_value=reset_value) + else: + paginator_strategy.reset(reset_value=reset_value) + assert paginator_strategy.initial_token == expected_initial_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 0fed6be46cc8..17ca98bc3019 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -1,15 +1,19 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import json from unittest.mock import MagicMock, Mock, patch import pytest import requests from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, DeclarativeCursor +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, DeclarativeCursor, ResumableFullRefreshCursor from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import CursorPaginationStrategy, PageIncrement from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever, SimpleRetrieverTestReadDecorator @@ -137,6 +141,227 @@ def test_simple_retriever_with_request_response_logs(mock_http_stream): assert actual_messages[3] == records[1] +@pytest.mark.parametrize( + "initial_state, expected_reset_value, expected_next_page", + [ + pytest.param(None, None, 1, id="test_initial_sync_no_state"), + pytest.param({"next_page_token": 10}, 10, 11, id="test_reset_with_next_page_token"), + ] +) +def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_state, expected_reset_value, expected_next_page): + expected_records = [ + Record(data={"id": "abc"}, associated_slice=None), + Record(data={"id": "def"}, associated_slice=None), + Record(data={"id": "ghi"}, associated_slice=None), + Record(data={"id": "jkl"}, associated_slice=None), + Record(data={"id": "mno"}, associated_slice=None), + Record(data={"id": "123"}, associated_slice=None), + Record(data={"id": "456"}, associated_slice=None), + Record(data={"id": "789"}, associated_slice=None), + ] + + response = requests.Response() + response.status_code = 200 + response._content = json.dumps({"data": [record.data for record in expected_records[:5]]}).encode("utf-8") + + requester = MagicMock() + requester.send_request.side_effect = [ + response, + response, + ] + + record_selector = MagicMock() + record_selector.select_records.side_effect = [ + [ + expected_records[0], + expected_records[1], + expected_records[2], + expected_records[3], + expected_records[4], + ], + [ + expected_records[5], + expected_records[6], + expected_records[7], + ] + ] + + page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) + paginator = DefaultPaginator(config={}, pagination_strategy=page_increment_strategy, url_base="https://airbyte.io", parameters={}) + paginator.reset = Mock(wraps=paginator.reset) + + stream_slicer = ResumableFullRefreshCursor(parameters={}) + if initial_state: + stream_slicer.set_initial_state(initial_state) + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + paginator=paginator, + record_selector=record_selector, + stream_slicer=stream_slicer, + cursor=stream_slicer, + parameters={}, + config={}, + ) + + stream_slice = list(stream_slicer.stream_slices())[0] + actual_records = [r for r in retriever.read_records(records_schema={}, stream_slice=stream_slice)] + + assert len(actual_records) == 5 + assert actual_records == expected_records[:5] + assert retriever.state == {"next_page_token": expected_next_page} + + actual_records = [r for r in retriever.read_records(records_schema={}, stream_slice=stream_slice)] + assert len(actual_records) == 3 + assert actual_records == expected_records[5:] + assert retriever.state == {"__ab_full_refresh_sync_complete": True} + + paginator.reset.assert_called_once_with(reset_value=expected_reset_value) + + +@pytest.mark.parametrize( + "initial_state, expected_reset_value, expected_next_page", + [ + pytest.param(None, None, 1, id="test_initial_sync_no_state"), + pytest.param( + {"next_page_token": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens"}, + "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens", + "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens", + id="test_reset_with_next_page_token" + ), + ] +) +def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination(initial_state, expected_reset_value, expected_next_page): + expected_records = [ + Record(data={"name": "ed_baldwin"}, associated_slice=None), + Record(data={"name": "danielle_poole"}, associated_slice=None), + Record(data={"name": "tracy_stevens"}, associated_slice=None), + Record(data={"name": "deke_slayton"}, associated_slice=None), + Record(data={"name": "molly_cobb"}, associated_slice=None), + Record(data={"name": "gordo_stevens"}, associated_slice=None), + Record(data={"name": "margo_madison"}, associated_slice=None), + Record(data={"name": "ellen_waverly"}, associated_slice=None), + ] + + response_1 = requests.Response() + response_1.status_code = 200 + response_body = { + "data": [r.data for r in expected_records[:5]], + "next_page": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens" + } + response_1._content = json.dumps(response_body).encode("utf-8") + response_2 = requests.Response() + response_2.status_code = 200 + response_body = { + "data": [r.data for r in expected_records[5:]], + } + response_2._content = json.dumps(response_body).encode("utf-8") + + requester = MagicMock() + requester.send_request.side_effect = [ + response_1, + response_2, + ] + + record_selector = MagicMock() + record_selector.select_records.side_effect = [ + [ + expected_records[0], + expected_records[1], + expected_records[2], + expected_records[3], + expected_records[4], + ], + [ + expected_records[5], + expected_records[6], + expected_records[7], + ] + ] + + cursor_pagination_strategy = CursorPaginationStrategy(config={}, cursor_value="{{ response.next_page }}", parameters={}) + paginator = DefaultPaginator(config={}, pagination_strategy=cursor_pagination_strategy, url_base="https://for-all-mankind.nasa.com/api/v1", parameters={}) + paginator.reset = Mock(wraps=paginator.reset) + + stream_slicer = ResumableFullRefreshCursor(parameters={}) + if initial_state: + stream_slicer.set_initial_state(initial_state) + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + paginator=paginator, + record_selector=record_selector, + stream_slicer=stream_slicer, + cursor=stream_slicer, + parameters={}, + config={}, + ) + + stream_slice = list(stream_slicer.stream_slices())[0] + actual_records = [r for r in retriever.read_records(records_schema={}, stream_slice=stream_slice)] + + assert len(actual_records) == 5 + assert actual_records == expected_records[:5] + assert retriever.state == {"next_page_token": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens"} + + actual_records = [r for r in retriever.read_records(records_schema={}, stream_slice=stream_slice)] + assert len(actual_records) == 3 + assert actual_records == expected_records[5:] + assert retriever.state == {"__ab_full_refresh_sync_complete": True} + + paginator.reset.assert_called_once_with(reset_value=expected_reset_value) + + +def test_simple_retriever_resumable_full_refresh_cursor_reset_skip_completed_stream(): + expected_records = [ + Record(data={"id": "abc"}, associated_slice=None), + Record(data={"id": "def"}, associated_slice=None), + ] + + response = requests.Response() + response.status_code = 200 + response._content = json.dumps({}).encode("utf-8") + + requester = MagicMock() + requester.send_request.side_effect = [ + response, + ] + + record_selector = MagicMock() + record_selector.select_records.return_value = [expected_records[0],expected_records[1],] + + page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) + paginator = DefaultPaginator(config={}, pagination_strategy=page_increment_strategy, url_base="https://airbyte.io", parameters={}) + paginator.reset = Mock(wraps=paginator.reset) + + stream_slicer = ResumableFullRefreshCursor(parameters={}) + stream_slicer.set_initial_state({"__ab_full_refresh_sync_complete": True}) + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + paginator=paginator, + record_selector=record_selector, + stream_slicer=stream_slicer, + cursor=stream_slicer, + parameters={}, + config={}, + ) + + stream_slice = list(stream_slicer.stream_slices())[0] + actual_records = [r for r in retriever.read_records(records_schema={}, stream_slice=stream_slice)] + + assert len(actual_records) == 0 + assert retriever.state == {"__ab_full_refresh_sync_complete": True} + + paginator.reset.assert_not_called() + + @pytest.mark.parametrize( "test_name, paginator_mapping, stream_slicer_mapping, expected_mapping", [ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index 8a1026776fd3..27a49ebb3067 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -837,7 +837,7 @@ def _create_page(response_body): ) * 10, [{"ABC": 0}, {"AED": 1}], - [call({}, {}, None)], + [call({}, {})], ), ( "test_read_manifest_with_added_fields", @@ -906,7 +906,7 @@ def _create_page(response_body): ) * 10, [{"ABC": 0, "added_field_key": "added_field_value"}, {"AED": 1, "added_field_key": "added_field_value"}], - [call({}, {}, None)], + [call({}, {})], ), ( "test_read_with_pagination_no_partitions", @@ -980,7 +980,7 @@ def _create_page(response_body): ) * 10, [{"ABC": 0}, {"AED": 1}, {"USD": 2}], - [call({}, {}, None), call({}, {}, {"next_page_token": "next"})], + [call({}, {}), call({"next_page_token": "next"}, {"next_page_token": "next"})], ), ( "test_no_pagination_with_partition_router", diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py index f1cdac5838b2..ae00e274764b 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py @@ -255,6 +255,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -354,6 +355,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -462,6 +464,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -615,6 +618,7 @@ "name": "songs_stream", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -638,6 +642,7 @@ "name": "festivals_stream", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -737,6 +742,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py index 3c93e046e87d..0b662519f276 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py @@ -86,6 +86,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -166,6 +167,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -264,6 +266,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -323,6 +326,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -438,6 +442,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -534,6 +539,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -660,6 +666,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -795,6 +802,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -952,6 +960,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1098,6 +1107,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1225,6 +1235,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1352,6 +1363,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1533,6 +1545,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1723,6 +1736,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1905,6 +1919,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2087,6 +2102,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2203,6 +2219,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2320,6 +2337,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2458,6 +2476,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2598,6 +2617,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2762,6 +2782,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index 5ef717647e5b..6a31db2be329 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -445,6 +445,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -535,6 +536,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -551,6 +553,7 @@ "name": "stream2", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -659,6 +662,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -764,6 +768,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -858,6 +863,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -942,6 +948,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "json_schema": { @@ -956,6 +963,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -1032,6 +1040,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], } @@ -1123,6 +1132,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -1139,6 +1149,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -1251,6 +1262,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1369,6 +1381,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -1387,6 +1400,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -1488,6 +1502,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1551,6 +1566,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1653,6 +1669,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -1669,6 +1686,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -1761,6 +1779,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1833,6 +1852,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -1849,6 +1869,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -1911,6 +1932,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1983,6 +2005,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2052,6 +2075,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2119,6 +2143,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2188,6 +2213,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2257,6 +2283,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2340,6 +2367,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2413,6 +2441,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2481,6 +2510,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2554,6 +2584,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2623,6 +2654,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2692,6 +2724,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2765,6 +2798,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2834,6 +2868,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2906,6 +2941,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -2977,6 +3013,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -3045,6 +3082,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -3101,6 +3139,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -3146,6 +3185,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py index df3f76a497d5..95cde6b73a48 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py @@ -85,6 +85,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -164,6 +165,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -261,6 +263,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -320,6 +323,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -435,6 +439,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -531,6 +536,7 @@ "default_cursor_field": ["_ab_source_file_last_modified"], "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "json_schema": { "type": "object", "properties": { @@ -656,6 +662,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -791,6 +798,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -948,6 +956,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1096,6 +1105,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1222,6 +1232,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1411,6 +1422,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1593,6 +1605,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1708,6 +1721,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -1845,6 +1859,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py index cee4c5b9a1e2..23e879306e4c 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py @@ -64,6 +64,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -158,6 +159,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -269,6 +271,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -380,6 +383,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -480,6 +484,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -565,6 +570,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -585,6 +591,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -688,6 +695,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -793,6 +801,7 @@ }, "name": "stream1", "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], }, @@ -813,6 +822,7 @@ "source_defined_cursor": True, "default_cursor_field": ["_ab_source_file_last_modified"], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -901,6 +911,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py index 30ffa263f88e..732660e564d8 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py @@ -208,6 +208,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -279,6 +280,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -323,6 +325,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -468,6 +471,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -555,6 +559,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -609,6 +614,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -659,6 +665,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -711,6 +718,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -747,6 +755,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py index da1e468c9df5..4257da83e604 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py @@ -77,6 +77,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -159,6 +160,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -227,6 +229,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -277,6 +280,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -341,6 +345,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -421,6 +426,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -499,6 +505,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -567,6 +574,7 @@ "source_defined_cursor": True, "source_defined_primary_key": [["document_key"]], "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py index 974bbf558974..e83ab5345787 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -53,6 +53,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -161,6 +162,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -203,6 +205,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -284,6 +287,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -306,6 +310,7 @@ "name": "stream2", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -322,6 +327,7 @@ "name": "stream3", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -502,6 +508,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -518,6 +525,7 @@ "name": "stream2", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -534,6 +542,7 @@ "name": "stream3", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } @@ -653,6 +662,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -669,6 +679,7 @@ "name": "stream2", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "default_cursor_field": ["_ab_source_file_last_modified"], @@ -685,6 +696,7 @@ "name": "stream3", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py index 4ff096954523..47e37dffd9c4 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py @@ -70,6 +70,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, } ] } @@ -164,6 +165,7 @@ "name": "stream1", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, { "json_schema": { @@ -183,6 +185,7 @@ "name": "stream2", "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], + "is_resumable": True, }, ] } diff --git a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py index 5c6015d40f35..ea9849aff9ed 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py @@ -1,10 +1,15 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from unittest.mock import Mock + +import pytest from airbyte_cdk.sources.streams.checkpoint import ( + CursorBasedCheckpointReader, FullRefreshCheckpointReader, IncrementalCheckpointReader, ResumableFullRefreshCheckpointReader, ) +from airbyte_cdk.sources.types import StreamSlice def test_incremental_checkpoint_reader_next_slice(): @@ -87,3 +92,124 @@ def test_full_refresh_checkpoint_reader_substream(): assert checkpoint_reader.get_checkpoint() is None assert checkpoint_reader.next() is None assert checkpoint_reader.get_checkpoint() == {"__ab_no_cursor_state_message": True} + + +def test_cursor_based_checkpoint_reader_incremental(): + expected_slices = [ + StreamSlice(cursor_slice={"start_date": "2024-01-01", "end_date": "2024-02-01"}, partition={}), + StreamSlice(cursor_slice={"start_date": "2024-02-01", "end_date": "2024-03-01"}, partition={}), + StreamSlice(cursor_slice={"start_date": "2024-03-01", "end_date": "2024-04-01"}, partition={}), + ] + + expected_stream_state = {"end_date": "2024-02-01"} + + incremental_cursor = Mock() + incremental_cursor.stream_slices.return_value = expected_slices + incremental_cursor.get_stream_state.return_value = expected_stream_state + + checkpoint_reader = CursorBasedCheckpointReader( + cursor=incremental_cursor, stream_slices=incremental_cursor.stream_slices(), read_state_from_cursor=False + ) + + assert checkpoint_reader.next() == expected_slices[0] + actual_state = checkpoint_reader.get_checkpoint() + assert actual_state == expected_stream_state + assert checkpoint_reader.next() == expected_slices[1] + assert checkpoint_reader.next() == expected_slices[2] + finished = checkpoint_reader.next() + assert finished is None + + # A finished checkpoint_reader should return None for the final checkpoint to avoid emitting duplicate state + assert checkpoint_reader.get_checkpoint() is None + + +def test_cursor_based_checkpoint_reader_resumable_full_refresh(): + expected_slices = [ + StreamSlice(cursor_slice={}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 2}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 3}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 4}, partition={}), + StreamSlice(cursor_slice={"__ab_full_refresh_sync_complete": True}, partition={}), + ] + + expected_stream_state = {"next_page_token": 2} + + rfr_cursor = Mock() + rfr_cursor.stream_slices.return_value = [StreamSlice(cursor_slice={}, partition={})] + rfr_cursor.select_state.side_effect = expected_slices[1:] + rfr_cursor.get_stream_state.return_value = expected_stream_state + + checkpoint_reader = CursorBasedCheckpointReader( + cursor=rfr_cursor, stream_slices=rfr_cursor.stream_slices(), read_state_from_cursor=True + ) + + assert checkpoint_reader.next() == expected_slices[0] + actual_state = checkpoint_reader.get_checkpoint() + assert actual_state == expected_stream_state + assert checkpoint_reader.next() == expected_slices[1] + assert checkpoint_reader.next() == expected_slices[2] + assert checkpoint_reader.next() == expected_slices[3] + finished = checkpoint_reader.next() + assert finished is None + + # A finished checkpoint_reader should return None for the final checkpoint to avoid emitting duplicate state + assert checkpoint_reader.get_checkpoint() is None + + +def test_cursor_based_checkpoint_reader_resumable_full_refresh_parents(): + expected_slices = [ + StreamSlice(cursor_slice={"start_date": "2024-01-01", "end_date": "2024-02-01"}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 2}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 3}, partition={}), + StreamSlice(cursor_slice={"start_date": "2024-02-01", "end_date": "2024-03-01"}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 2}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 3}, partition={}), + ] + + expected_stream_state = {"next_page_token": 2} + + rfr_cursor = Mock() + rfr_cursor.stream_slices.return_value = [ + StreamSlice(cursor_slice={"start_date": "2024-01-01", "end_date": "2024-02-01"}, partition={}), + StreamSlice(cursor_slice={"start_date": "2024-02-01", "end_date": "2024-03-01"}, partition={}), + ] + rfr_cursor.select_state.side_effect = [ + StreamSlice(cursor_slice={"next_page_token": 2}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 3}, partition={}), + StreamSlice(cursor_slice={"__ab_full_refresh_sync_complete": True}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 2}, partition={}), + StreamSlice(cursor_slice={"next_page_token": 3}, partition={}), + StreamSlice(cursor_slice={"__ab_full_refresh_sync_complete": True}, partition={}), + ] + rfr_cursor.get_stream_state.return_value = expected_stream_state + + checkpoint_reader = CursorBasedCheckpointReader( + cursor=rfr_cursor, stream_slices=rfr_cursor.stream_slices(), read_state_from_cursor=True + ) + + assert checkpoint_reader.next() == expected_slices[0] + actual_state = checkpoint_reader.get_checkpoint() + assert actual_state == expected_stream_state + assert checkpoint_reader.next() == expected_slices[1] + assert checkpoint_reader.next() == expected_slices[2] + assert checkpoint_reader.next() == expected_slices[3] + assert checkpoint_reader.next() == expected_slices[4] + assert checkpoint_reader.next() == expected_slices[5] + finished = checkpoint_reader.next() + assert finished is None + + # A finished checkpoint_reader should return None for the final checkpoint to avoid emitting duplicate state + assert checkpoint_reader.get_checkpoint() is None + + +def test_cursor_based_checkpoint_reader_resumable_full_refresh_invalid_slice(): + rfr_cursor = Mock() + rfr_cursor.stream_slices.return_value = [{"invalid": "stream_slice"}] + rfr_cursor.select_state.side_effect = [StreamSlice(cursor_slice={"invalid": "stream_slice"}, partition={})] + + checkpoint_reader = CursorBasedCheckpointReader( + cursor=rfr_cursor, stream_slices=rfr_cursor.stream_slices(), read_state_from_cursor=True + ) + + with pytest.raises(ValueError): + checkpoint_reader.next() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py index 5ec17253da4f..2cb40ca4910f 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py @@ -193,8 +193,8 @@ def test_as_airbyte_stream_full_refresh(mocker): mocker.patch.object(StreamStubFullRefresh, "get_json_schema", return_value={}) airbyte_stream = test_stream.as_airbyte_stream() - exp = AirbyteStream(name="stream_stub_full_refresh", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]) - assert exp == airbyte_stream + exp = AirbyteStream(name="stream_stub_full_refresh", json_schema={}, supported_sync_modes=[SyncMode.full_refresh], is_resumable=False) + assert airbyte_stream == exp def test_as_airbyte_stream_incremental(mocker): @@ -215,8 +215,9 @@ def test_as_airbyte_stream_incremental(mocker): default_cursor_field=["test_cursor"], source_defined_cursor=True, source_defined_primary_key=[["primary_key"]], + is_resumable=True, ) - assert exp == airbyte_stream + assert airbyte_stream == exp def test_supports_incremental_cursor_set(): @@ -266,8 +267,9 @@ def test_namespace_set_to_empty_string(mocker): source_defined_cursor=True, source_defined_primary_key=[["primary_key"]], namespace=None, + is_resumable=True, ) - assert exp == airbyte_stream + assert airbyte_stream == exp def test_namespace_not_set():