From 52bca997e170644dcf4ba2e394e5c102b9cb6447 Mon Sep 17 00:00:00 2001 From: Andrey Vasnetsov Date: Tue, 26 Jul 2022 10:05:16 +0200 Subject: [PATCH] V0.8.5 0.8.7 (#54) * upd to support v0.8.5 of Qdrant * add generated files --- pyproject.toml | 2 +- qdrant_client/grpc/__init__.py | 60 ++++ qdrant_client/grpc/snapshots_service.proto | 17 +- qdrant_client/http/api/cluster_api.py | 46 ++- qdrant_client/http/api/collections_api.py | 100 ++++-- qdrant_client/http/api/default_api.py | 198 +++++++++++ qdrant_client/http/api/points_api.py | 66 ++-- qdrant_client/http/api/snapshots_api.py | 380 +++++++++++++++++++++ qdrant_client/http/api_client.py | 6 + qdrant_client/http/models/models.py | 291 +++++++++++++++- qdrant_client/qdrant_client.py | 16 + tests/integration-tests.sh | 2 +- 12 files changed, 1102 insertions(+), 82 deletions(-) create mode 100644 qdrant_client/http/api/default_api.py create mode 100644 qdrant_client/http/api/snapshots_api.py diff --git a/pyproject.toml b/pyproject.toml index 6db2acb2..6b502fea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "qdrant_client" -version = "0.8.6" +version = "0.8.7" description = "Client library for the Qdrant vector search engine" authors = ["Andrey Vasnetsov "] packages = [ diff --git a/qdrant_client/grpc/__init__.py b/qdrant_client/grpc/__init__.py index 9b915767..81ff3805 100644 --- a/qdrant_client/grpc/__init__.py +++ b/qdrant_client/grpc/__init__.py @@ -677,6 +677,16 @@ class GeoPoint(betterproto.Message): lat: float = betterproto.double_field(2) +@dataclass(eq=False, repr=False) +class CreateFullSnapshotRequest(betterproto.Message): + pass + + +@dataclass(eq=False, repr=False) +class ListFullSnapshotsRequest(betterproto.Message): + pass + + @dataclass(eq=False, repr=False) class CreateSnapshotRequest(betterproto.Message): collection_name: str = betterproto.string_field(1) @@ -1099,6 +1109,22 @@ async def list(self, *, collection_name: str = "") -> "ListSnapshotsResponse": "/qdrant.Snapshots/List", request, ListSnapshotsResponse ) + async def create_full(self) -> "CreateSnapshotResponse": + + request = CreateFullSnapshotRequest() + + return await self._unary_unary( + "/qdrant.Snapshots/CreateFull", request, CreateSnapshotResponse + ) + + async def list_full(self) -> "ListSnapshotsResponse": + + request = ListFullSnapshotsRequest() + + return await self._unary_unary( + "/qdrant.Snapshots/ListFull", request, ListSnapshotsResponse + ) + class QdrantStub(betterproto.ServiceStub): async def health_check(self) -> "HealthCheckReply": @@ -1613,6 +1639,12 @@ async def create(self, collection_name: str) -> "CreateSnapshotResponse": async def list(self, collection_name: str) -> "ListSnapshotsResponse": raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + async def create_full(self) -> "CreateSnapshotResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + + async def list_full(self) -> "ListSnapshotsResponse": + raise grpclib.GRPCError(grpclib.const.Status.UNIMPLEMENTED) + async def __rpc_create(self, stream: grpclib.server.Stream) -> None: request = await stream.recv_message() @@ -1633,6 +1665,22 @@ async def __rpc_list(self, stream: grpclib.server.Stream) -> None: response = await self.list(**request_kwargs) await stream.send_message(response) + async def __rpc_create_full(self, stream: grpclib.server.Stream) -> None: + request = await stream.recv_message() + + request_kwargs = {} + + response = await self.create_full(**request_kwargs) + await stream.send_message(response) + + async def __rpc_list_full(self, stream: grpclib.server.Stream) -> None: + request = await stream.recv_message() + + request_kwargs = {} + + response = await self.list_full(**request_kwargs) + await stream.send_message(response) + def __mapping__(self) -> Dict[str, grpclib.const.Handler]: return { "/qdrant.Snapshots/Create": grpclib.const.Handler( @@ -1647,6 +1695,18 @@ def __mapping__(self) -> Dict[str, grpclib.const.Handler]: ListSnapshotsRequest, ListSnapshotsResponse, ), + "/qdrant.Snapshots/CreateFull": grpclib.const.Handler( + self.__rpc_create_full, + grpclib.const.Cardinality.UNARY_UNARY, + CreateFullSnapshotRequest, + CreateSnapshotResponse, + ), + "/qdrant.Snapshots/ListFull": grpclib.const.Handler( + self.__rpc_list_full, + grpclib.const.Cardinality.UNARY_UNARY, + ListFullSnapshotsRequest, + ListSnapshotsResponse, + ), } diff --git a/qdrant_client/grpc/snapshots_service.proto b/qdrant_client/grpc/snapshots_service.proto index 2fd3cf2f..4dfffafd 100644 --- a/qdrant_client/grpc/snapshots_service.proto +++ b/qdrant_client/grpc/snapshots_service.proto @@ -7,15 +7,28 @@ import "google/protobuf/timestamp.proto"; service Snapshots { /* - Create snapshot + Create collection snapshot */ rpc Create (CreateSnapshotRequest) returns (CreateSnapshotResponse) {} /* - List snapshots + List collection snapshots */ rpc List (ListSnapshotsRequest) returns (ListSnapshotsResponse) {} + /* + Create full storage snapshot + */ + rpc CreateFull (CreateFullSnapshotRequest) returns (CreateSnapshotResponse) {} + /* + List full storage snapshots + */ + rpc ListFull (ListFullSnapshotsRequest) returns (ListSnapshotsResponse) {} + } +message CreateFullSnapshotRequest {} + +message ListFullSnapshotsRequest {} + message CreateSnapshotRequest { string collection_name = 1; // Name of the collection } diff --git a/qdrant_client/http/api/cluster_api.py b/qdrant_client/http/api/cluster_api.py index 8138c739..df9040c8 100644 --- a/qdrant_client/http/api/cluster_api.py +++ b/qdrant_client/http/api/cluster_api.py @@ -160,27 +160,67 @@ def _build_for_cluster_status( Get information about the current state and composition of the cluster """ return self.api_client.request( - type_=m.InlineResponse200, + type_=m.InlineResponse2001, method="GET", url="/cluster", ) + def _build_for_collection_cluster_info( + self, + collection_name: str, + ): + """ + Get cluster information for a collection + """ + path_params = { + "collection_name": str(collection_name), + } + + return self.api_client.request( + type_=m.InlineResponse2006, + method="GET", + url="/collections/{collection_name}/cluster", + path_params=path_params, + ) + class AsyncClusterApi(_ClusterApi): async def cluster_status( self, - ) -> m.InlineResponse200: + ) -> m.InlineResponse2001: """ Get information about the current state and composition of the cluster """ return await self._build_for_cluster_status() + async def collection_cluster_info( + self, + collection_name: str, + ) -> m.InlineResponse2006: + """ + Get cluster information for a collection + """ + return await self._build_for_collection_cluster_info( + collection_name=collection_name, + ) + class SyncClusterApi(_ClusterApi): def cluster_status( self, - ) -> m.InlineResponse200: + ) -> m.InlineResponse2001: """ Get information about the current state and composition of the cluster """ return self._build_for_cluster_status() + + def collection_cluster_info( + self, + collection_name: str, + ) -> m.InlineResponse2006: + """ + Get cluster information for a collection + """ + return self._build_for_collection_cluster_info( + collection_name=collection_name, + ) diff --git a/qdrant_client/http/api/collections_api.py b/qdrant_client/http/api/collections_api.py index db2970f9..bd697799 100644 --- a/qdrant_client/http/api/collections_api.py +++ b/qdrant_client/http/api/collections_api.py @@ -153,6 +153,24 @@ class _CollectionsApi: def __init__(self, api_client: "Union[ApiClient, AsyncApiClient]"): self.api_client = api_client + def _build_for_collection_cluster_info( + self, + collection_name: str, + ): + """ + Get cluster information for a collection + """ + path_params = { + "collection_name": str(collection_name), + } + + return self.api_client.request( + type_=m.InlineResponse2006, + method="GET", + url="/collections/{collection_name}/cluster", + path_params=path_params, + ) + def _build_for_create_collection( self, collection_name: str, @@ -173,7 +191,7 @@ def _build_for_create_collection( body = jsonable_encoder(create_collection) return self.api_client.request( - type_=m.InlineResponse2003, + type_=m.InlineResponse2004, method="PUT", url="/collections/{collection_name}", path_params=path_params, @@ -201,7 +219,7 @@ def _build_for_create_field_index( body = jsonable_encoder(create_field_index) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="PUT", url="/collections/{collection_name}/index", path_params=path_params, @@ -221,7 +239,7 @@ def _build_for_create_snapshot( } return self.api_client.request( - type_=m.InlineResponse2006, + type_=m.InlineResponse2008, method="POST", url="/collections/{collection_name}/snapshots", path_params=path_params, @@ -244,7 +262,7 @@ def _build_for_delete_collection( query_params["timeout"] = str(timeout) return self.api_client.request( - type_=m.InlineResponse2003, + type_=m.InlineResponse2004, method="DELETE", url="/collections/{collection_name}", path_params=path_params, @@ -270,7 +288,7 @@ def _build_for_delete_field_index( query_params["wait"] = str(wait).lower() return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="DELETE", url="/collections/{collection_name}/index/{field_name}", path_params=path_params, @@ -289,7 +307,7 @@ def _build_for_get_collection( } return self.api_client.request( - type_=m.InlineResponse2002, + type_=m.InlineResponse2003, method="GET", url="/collections/{collection_name}", path_params=path_params, @@ -302,7 +320,7 @@ def _build_for_get_collections( Get list name of all existing collections """ return self.api_client.request( - type_=m.InlineResponse2001, + type_=m.InlineResponse2002, method="GET", url="/collections", ) @@ -339,7 +357,7 @@ def _build_for_list_snapshots( } return self.api_client.request( - type_=m.InlineResponse2005, + type_=m.InlineResponse2007, method="GET", url="/collections/{collection_name}/snapshots", path_params=path_params, @@ -357,7 +375,7 @@ def _build_for_update_aliases( body = jsonable_encoder(change_aliases_operation) return self.api_client.request( - type_=m.InlineResponse2003, method="POST", url="/collections/aliases", params=query_params, json=body + type_=m.InlineResponse2004, method="POST", url="/collections/aliases", params=query_params, json=body ) def _build_for_update_collection( @@ -380,7 +398,7 @@ def _build_for_update_collection( body = jsonable_encoder(update_collection) return self.api_client.request( - type_=m.InlineResponse2003, + type_=m.InlineResponse2004, method="PATCH", url="/collections/{collection_name}", path_params=path_params, @@ -390,12 +408,23 @@ def _build_for_update_collection( class AsyncCollectionsApi(_CollectionsApi): + async def collection_cluster_info( + self, + collection_name: str, + ) -> m.InlineResponse2006: + """ + Get cluster information for a collection + """ + return await self._build_for_collection_cluster_info( + collection_name=collection_name, + ) + async def create_collection( self, collection_name: str, timeout: int = None, create_collection: m.CreateCollection = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Create new collection with given parameters """ @@ -410,7 +439,7 @@ async def create_field_index( collection_name: str, wait: bool = None, create_field_index: m.CreateFieldIndex = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Create index for field in collection """ @@ -423,7 +452,7 @@ async def create_field_index( async def create_snapshot( self, collection_name: str, - ) -> m.InlineResponse2006: + ) -> m.InlineResponse2008: """ Create new snapshot for a collection """ @@ -435,7 +464,7 @@ async def delete_collection( self, collection_name: str, timeout: int = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Drop collection and all associated data """ @@ -449,7 +478,7 @@ async def delete_field_index( collection_name: str, field_name: str, wait: bool = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete field index for collection """ @@ -462,7 +491,7 @@ async def delete_field_index( async def get_collection( self, collection_name: str, - ) -> m.InlineResponse2002: + ) -> m.InlineResponse2003: """ Get detailed information about specified existing collection """ @@ -472,7 +501,7 @@ async def get_collection( async def get_collections( self, - ) -> m.InlineResponse2001: + ) -> m.InlineResponse2002: """ Get list name of all existing collections """ @@ -494,7 +523,7 @@ async def get_snapshot( async def list_snapshots( self, collection_name: str, - ) -> m.InlineResponse2005: + ) -> m.InlineResponse2007: """ Get list of snapshots for a collection """ @@ -506,7 +535,7 @@ async def update_aliases( self, timeout: int = None, change_aliases_operation: m.ChangeAliasesOperation = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: return await self._build_for_update_aliases( timeout=timeout, change_aliases_operation=change_aliases_operation, @@ -517,7 +546,7 @@ async def update_collection( collection_name: str, timeout: int = None, update_collection: m.UpdateCollection = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Update parameters of the existing collection """ @@ -529,12 +558,23 @@ async def update_collection( class SyncCollectionsApi(_CollectionsApi): + def collection_cluster_info( + self, + collection_name: str, + ) -> m.InlineResponse2006: + """ + Get cluster information for a collection + """ + return self._build_for_collection_cluster_info( + collection_name=collection_name, + ) + def create_collection( self, collection_name: str, timeout: int = None, create_collection: m.CreateCollection = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Create new collection with given parameters """ @@ -549,7 +589,7 @@ def create_field_index( collection_name: str, wait: bool = None, create_field_index: m.CreateFieldIndex = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Create index for field in collection """ @@ -562,7 +602,7 @@ def create_field_index( def create_snapshot( self, collection_name: str, - ) -> m.InlineResponse2006: + ) -> m.InlineResponse2008: """ Create new snapshot for a collection """ @@ -574,7 +614,7 @@ def delete_collection( self, collection_name: str, timeout: int = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Drop collection and all associated data """ @@ -588,7 +628,7 @@ def delete_field_index( collection_name: str, field_name: str, wait: bool = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete field index for collection """ @@ -601,7 +641,7 @@ def delete_field_index( def get_collection( self, collection_name: str, - ) -> m.InlineResponse2002: + ) -> m.InlineResponse2003: """ Get detailed information about specified existing collection """ @@ -611,7 +651,7 @@ def get_collection( def get_collections( self, - ) -> m.InlineResponse2001: + ) -> m.InlineResponse2002: """ Get list name of all existing collections """ @@ -633,7 +673,7 @@ def get_snapshot( def list_snapshots( self, collection_name: str, - ) -> m.InlineResponse2005: + ) -> m.InlineResponse2007: """ Get list of snapshots for a collection """ @@ -645,7 +685,7 @@ def update_aliases( self, timeout: int = None, change_aliases_operation: m.ChangeAliasesOperation = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: return self._build_for_update_aliases( timeout=timeout, change_aliases_operation=change_aliases_operation, @@ -656,7 +696,7 @@ def update_collection( collection_name: str, timeout: int = None, update_collection: m.UpdateCollection = None, - ) -> m.InlineResponse2003: + ) -> m.InlineResponse2004: """ Update parameters of the existing collection """ diff --git a/qdrant_client/http/api/default_api.py b/qdrant_client/http/api/default_api.py new file mode 100644 index 00000000..bb95150c --- /dev/null +++ b/qdrant_client/http/api/default_api.py @@ -0,0 +1,198 @@ +# flake8: noqa E501 +from enum import Enum +from pathlib import PurePath +from types import GeneratorType +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Set, Tuple, Union + +from pydantic.json import ENCODERS_BY_TYPE +from pydantic.main import BaseModel +from qdrant_client.http.models import models as m + +SetIntStr = Set[Union[int, str]] +DictIntStrAny = Dict[Union[int, str], Any] +file = None + + +def generate_encoders_by_class_tuples(type_encoder_map: Dict[Any, Callable]) -> Dict[Callable, Tuple]: + encoders_by_classes: Dict[Callable, List] = {} + for type_, encoder in type_encoder_map.items(): + encoders_by_classes.setdefault(encoder, []).append(type_) + encoders_by_class_tuples: Dict[Callable, Tuple] = {} + for encoder, classes in encoders_by_classes.items(): + encoders_by_class_tuples[encoder] = tuple(classes) + return encoders_by_class_tuples + + +encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE) + + +def jsonable_encoder( + obj: Any, + include: Union[SetIntStr, DictIntStrAny] = None, + exclude=None, + by_alias: bool = True, + skip_defaults: bool = None, + exclude_unset: bool = False, + include_none: bool = True, + custom_encoder=None, + sqlalchemy_safe: bool = True, +) -> Any: + if exclude is None: + exclude = set() + if custom_encoder is None: + custom_encoder = {} + if include is not None and not isinstance(include, set): + include = set(include) + if exclude is not None and not isinstance(exclude, set): + exclude = set(exclude) + if isinstance(obj, BaseModel): + encoder = getattr(obj.Config, "json_encoders", {}) + if custom_encoder: + encoder.update(custom_encoder) + obj_dict = obj.dict( + include=include, + exclude=exclude, + by_alias=by_alias, + exclude_unset=bool(exclude_unset or skip_defaults), + ) + + return jsonable_encoder( + obj_dict, + include_none=include_none, + custom_encoder=encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, PurePath): + return str(obj) + if isinstance(obj, (str, int, float, type(None))): + return obj + if isinstance(obj, dict): + encoded_dict = {} + for key, value in obj.items(): + if ( + (not sqlalchemy_safe or (not isinstance(key, str)) or (not key.startswith("_sa"))) + and (value is not None or include_none) + and ((include and key in include) or key not in exclude) + ): + encoded_key = jsonable_encoder( + key, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_value = jsonable_encoder( + value, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_dict[encoded_key] = encoded_value + return encoded_dict + if isinstance(obj, (list, set, frozenset, GeneratorType, tuple)): + encoded_list = [] + for item in obj: + encoded_list.append( + jsonable_encoder( + item, + include=include, + exclude=exclude, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + ) + return encoded_list + + if custom_encoder: + if type(obj) in custom_encoder: + return custom_encoder[type(obj)](obj) + else: + for encoder_type, encoder in custom_encoder.items(): + if isinstance(obj, encoder_type): + return encoder(obj) + + if type(obj) in ENCODERS_BY_TYPE: + return ENCODERS_BY_TYPE[type(obj)](obj) + for encoder, classes_tuple in encoders_by_class_tuples.items(): + if isinstance(obj, classes_tuple): + return encoder(obj) + + errors: List[Exception] = [] + try: + data = dict(obj) + except Exception as e: + errors.append(e) + try: + data = vars(obj) + except Exception as e: + errors.append(e) + raise ValueError(errors) + return jsonable_encoder( + data, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + + +if TYPE_CHECKING: + from qdrant_client.http.api_client import ApiClient + + +class _DefaultApi: + def __init__(self, api_client: "Union[ApiClient, AsyncApiClient]"): + self.api_client = api_client + + def _build_for_telemetry( + self, + anonymize: bool = None, + ): + """ + Collect telemetry data including app info, system info, collections info, cluster info, configs and statistics + """ + query_params = {} + if anonymize is not None: + query_params["anonymize"] = str(anonymize).lower() + + return self.api_client.request( + type_=m.InlineResponse200, + method="GET", + url="/telemetry", + params=query_params, + ) + + +class AsyncDefaultApi(_DefaultApi): + async def telemetry( + self, + anonymize: bool = None, + ) -> m.InlineResponse200: + """ + Collect telemetry data including app info, system info, collections info, cluster info, configs and statistics + """ + return await self._build_for_telemetry( + anonymize=anonymize, + ) + + +class SyncDefaultApi(_DefaultApi): + def telemetry( + self, + anonymize: bool = None, + ) -> m.InlineResponse200: + """ + Collect telemetry data including app info, system info, collections info, cluster info, configs and statistics + """ + return self._build_for_telemetry( + anonymize=anonymize, + ) diff --git a/qdrant_client/http/api/points_api.py b/qdrant_client/http/api/points_api.py index 6d24b11a..35a0c47b 100644 --- a/qdrant_client/http/api/points_api.py +++ b/qdrant_client/http/api/points_api.py @@ -173,7 +173,7 @@ def _build_for_clear_payload( body = jsonable_encoder(points_selector) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="POST", url="/collections/{collection_name}/points/payload/clear", path_params=path_params, @@ -196,7 +196,7 @@ def _build_for_count_points( body = jsonable_encoder(count_request) return self.api_client.request( - type_=m.InlineResponse20011, + type_=m.InlineResponse20013, method="POST", url="/collections/{collection_name}/points/count", path_params=path_params, @@ -223,7 +223,7 @@ def _build_for_delete_payload( body = jsonable_encoder(delete_payload) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="POST", url="/collections/{collection_name}/points/payload/delete", path_params=path_params, @@ -251,7 +251,7 @@ def _build_for_delete_points( body = jsonable_encoder(points_selector) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="POST", url="/collections/{collection_name}/points/delete", path_params=path_params, @@ -273,7 +273,7 @@ def _build_for_get_point( } return self.api_client.request( - type_=m.InlineResponse2007, + type_=m.InlineResponse2009, method="GET", url="/collections/{collection_name}/points/{id}", path_params=path_params, @@ -294,7 +294,7 @@ def _build_for_get_points( body = jsonable_encoder(point_request) return self.api_client.request( - type_=m.InlineResponse2008, + type_=m.InlineResponse20010, method="POST", url="/collections/{collection_name}/points", path_params=path_params, @@ -316,7 +316,7 @@ def _build_for_recommend_points( body = jsonable_encoder(recommend_request) return self.api_client.request( - type_=m.InlineResponse20010, + type_=m.InlineResponse20012, method="POST", url="/collections/{collection_name}/points/recommend", path_params=path_params, @@ -338,7 +338,7 @@ def _build_for_scroll_points( body = jsonable_encoder(scroll_request) return self.api_client.request( - type_=m.InlineResponse2009, + type_=m.InlineResponse20011, method="POST", url="/collections/{collection_name}/points/scroll", path_params=path_params, @@ -360,7 +360,7 @@ def _build_for_search_points( body = jsonable_encoder(search_request) return self.api_client.request( - type_=m.InlineResponse20010, + type_=m.InlineResponse20012, method="POST", url="/collections/{collection_name}/points/search", path_params=path_params, @@ -387,7 +387,7 @@ def _build_for_set_payload( body = jsonable_encoder(set_payload) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="POST", url="/collections/{collection_name}/points/payload", path_params=path_params, @@ -415,7 +415,7 @@ def _build_for_upsert_points( body = jsonable_encoder(point_insert_operations) return self.api_client.request( - type_=m.InlineResponse2004, + type_=m.InlineResponse2005, method="PUT", url="/collections/{collection_name}/points", path_params=path_params, @@ -430,7 +430,7 @@ async def clear_payload( collection_name: str, wait: bool = None, points_selector: m.PointsSelector = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Remove all payload for specified points """ @@ -444,7 +444,7 @@ async def count_points( self, collection_name: str, count_request: m.CountRequest = None, - ) -> m.InlineResponse20011: + ) -> m.InlineResponse20013: """ Count points which matches given filtering condition """ @@ -458,7 +458,7 @@ async def delete_payload( collection_name: str, wait: bool = None, delete_payload: m.DeletePayload = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete specified key payload for points """ @@ -473,7 +473,7 @@ async def delete_points( collection_name: str, wait: bool = None, points_selector: m.PointsSelector = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete points """ @@ -487,7 +487,7 @@ async def get_point( self, collection_name: str, id: m.ExtendedPointId, - ) -> m.InlineResponse2007: + ) -> m.InlineResponse2009: """ Retrieve full information of single point by id """ @@ -500,7 +500,7 @@ async def get_points( self, collection_name: str, point_request: m.PointRequest = None, - ) -> m.InlineResponse2008: + ) -> m.InlineResponse20010: """ Retrieve multiple points by specified IDs """ @@ -513,7 +513,7 @@ async def recommend_points( self, collection_name: str, recommend_request: m.RecommendRequest = None, - ) -> m.InlineResponse20010: + ) -> m.InlineResponse20012: """ Look for the points which are closer to stored positive examples and at the same time further to negative examples. """ @@ -526,7 +526,7 @@ async def scroll_points( self, collection_name: str, scroll_request: m.ScrollRequest = None, - ) -> m.InlineResponse2009: + ) -> m.InlineResponse20011: """ Scroll request - paginate over all points which matches given filtering condition """ @@ -539,7 +539,7 @@ async def search_points( self, collection_name: str, search_request: m.SearchRequest = None, - ) -> m.InlineResponse20010: + ) -> m.InlineResponse20012: """ Retrieve closest points based on vector similarity and given filtering conditions """ @@ -553,7 +553,7 @@ async def set_payload( collection_name: str, wait: bool = None, set_payload: m.SetPayload = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Set payload for points """ @@ -568,7 +568,7 @@ async def upsert_points( collection_name: str, wait: bool = None, point_insert_operations: m.PointInsertOperations = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Perform insert + updates on points. If point with given ID already exists - it will be overwritten. """ @@ -585,7 +585,7 @@ def clear_payload( collection_name: str, wait: bool = None, points_selector: m.PointsSelector = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Remove all payload for specified points """ @@ -599,7 +599,7 @@ def count_points( self, collection_name: str, count_request: m.CountRequest = None, - ) -> m.InlineResponse20011: + ) -> m.InlineResponse20013: """ Count points which matches given filtering condition """ @@ -613,7 +613,7 @@ def delete_payload( collection_name: str, wait: bool = None, delete_payload: m.DeletePayload = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete specified key payload for points """ @@ -628,7 +628,7 @@ def delete_points( collection_name: str, wait: bool = None, points_selector: m.PointsSelector = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Delete points """ @@ -642,7 +642,7 @@ def get_point( self, collection_name: str, id: m.ExtendedPointId, - ) -> m.InlineResponse2007: + ) -> m.InlineResponse2009: """ Retrieve full information of single point by id """ @@ -655,7 +655,7 @@ def get_points( self, collection_name: str, point_request: m.PointRequest = None, - ) -> m.InlineResponse2008: + ) -> m.InlineResponse20010: """ Retrieve multiple points by specified IDs """ @@ -668,7 +668,7 @@ def recommend_points( self, collection_name: str, recommend_request: m.RecommendRequest = None, - ) -> m.InlineResponse20010: + ) -> m.InlineResponse20012: """ Look for the points which are closer to stored positive examples and at the same time further to negative examples. """ @@ -681,7 +681,7 @@ def scroll_points( self, collection_name: str, scroll_request: m.ScrollRequest = None, - ) -> m.InlineResponse2009: + ) -> m.InlineResponse20011: """ Scroll request - paginate over all points which matches given filtering condition """ @@ -694,7 +694,7 @@ def search_points( self, collection_name: str, search_request: m.SearchRequest = None, - ) -> m.InlineResponse20010: + ) -> m.InlineResponse20012: """ Retrieve closest points based on vector similarity and given filtering conditions """ @@ -708,7 +708,7 @@ def set_payload( collection_name: str, wait: bool = None, set_payload: m.SetPayload = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Set payload for points """ @@ -723,7 +723,7 @@ def upsert_points( collection_name: str, wait: bool = None, point_insert_operations: m.PointInsertOperations = None, - ) -> m.InlineResponse2004: + ) -> m.InlineResponse2005: """ Perform insert + updates on points. If point with given ID already exists - it will be overwritten. """ diff --git a/qdrant_client/http/api/snapshots_api.py b/qdrant_client/http/api/snapshots_api.py new file mode 100644 index 00000000..2db78660 --- /dev/null +++ b/qdrant_client/http/api/snapshots_api.py @@ -0,0 +1,380 @@ +# flake8: noqa E501 +from enum import Enum +from pathlib import PurePath +from types import GeneratorType +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Set, Tuple, Union + +from pydantic.json import ENCODERS_BY_TYPE +from pydantic.main import BaseModel +from qdrant_client.http.models import models as m + +SetIntStr = Set[Union[int, str]] +DictIntStrAny = Dict[Union[int, str], Any] +file = None + + +def generate_encoders_by_class_tuples(type_encoder_map: Dict[Any, Callable]) -> Dict[Callable, Tuple]: + encoders_by_classes: Dict[Callable, List] = {} + for type_, encoder in type_encoder_map.items(): + encoders_by_classes.setdefault(encoder, []).append(type_) + encoders_by_class_tuples: Dict[Callable, Tuple] = {} + for encoder, classes in encoders_by_classes.items(): + encoders_by_class_tuples[encoder] = tuple(classes) + return encoders_by_class_tuples + + +encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE) + + +def jsonable_encoder( + obj: Any, + include: Union[SetIntStr, DictIntStrAny] = None, + exclude=None, + by_alias: bool = True, + skip_defaults: bool = None, + exclude_unset: bool = False, + include_none: bool = True, + custom_encoder=None, + sqlalchemy_safe: bool = True, +) -> Any: + if exclude is None: + exclude = set() + if custom_encoder is None: + custom_encoder = {} + if include is not None and not isinstance(include, set): + include = set(include) + if exclude is not None and not isinstance(exclude, set): + exclude = set(exclude) + if isinstance(obj, BaseModel): + encoder = getattr(obj.Config, "json_encoders", {}) + if custom_encoder: + encoder.update(custom_encoder) + obj_dict = obj.dict( + include=include, + exclude=exclude, + by_alias=by_alias, + exclude_unset=bool(exclude_unset or skip_defaults), + ) + + return jsonable_encoder( + obj_dict, + include_none=include_none, + custom_encoder=encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, PurePath): + return str(obj) + if isinstance(obj, (str, int, float, type(None))): + return obj + if isinstance(obj, dict): + encoded_dict = {} + for key, value in obj.items(): + if ( + (not sqlalchemy_safe or (not isinstance(key, str)) or (not key.startswith("_sa"))) + and (value is not None or include_none) + and ((include and key in include) or key not in exclude) + ): + encoded_key = jsonable_encoder( + key, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_value = jsonable_encoder( + value, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + encoded_dict[encoded_key] = encoded_value + return encoded_dict + if isinstance(obj, (list, set, frozenset, GeneratorType, tuple)): + encoded_list = [] + for item in obj: + encoded_list.append( + jsonable_encoder( + item, + include=include, + exclude=exclude, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + ) + return encoded_list + + if custom_encoder: + if type(obj) in custom_encoder: + return custom_encoder[type(obj)](obj) + else: + for encoder_type, encoder in custom_encoder.items(): + if isinstance(obj, encoder_type): + return encoder(obj) + + if type(obj) in ENCODERS_BY_TYPE: + return ENCODERS_BY_TYPE[type(obj)](obj) + for encoder, classes_tuple in encoders_by_class_tuples.items(): + if isinstance(obj, classes_tuple): + return encoder(obj) + + errors: List[Exception] = [] + try: + data = dict(obj) + except Exception as e: + errors.append(e) + try: + data = vars(obj) + except Exception as e: + errors.append(e) + raise ValueError(errors) + return jsonable_encoder( + data, + by_alias=by_alias, + exclude_unset=exclude_unset, + include_none=include_none, + custom_encoder=custom_encoder, + sqlalchemy_safe=sqlalchemy_safe, + ) + + +if TYPE_CHECKING: + from qdrant_client.http.api_client import ApiClient + + +class _SnapshotsApi: + def __init__(self, api_client: "Union[ApiClient, AsyncApiClient]"): + self.api_client = api_client + + def _build_for_create_full_snapshot( + self, + ): + """ + Create new snapshot of the whole storage + """ + return self.api_client.request( + type_=m.InlineResponse2008, + method="POST", + url="/snapshots", + ) + + def _build_for_create_snapshot( + self, + collection_name: str, + ): + """ + Create new snapshot for a collection + """ + path_params = { + "collection_name": str(collection_name), + } + + return self.api_client.request( + type_=m.InlineResponse2008, + method="POST", + url="/collections/{collection_name}/snapshots", + path_params=path_params, + ) + + def _build_for_get_full_snapshot( + self, + snapshot_name: str, + ): + """ + Download specified snapshot of the whole storage as a file + """ + path_params = { + "snapshot_name": str(snapshot_name), + } + + return self.api_client.request( + type_=file, + method="GET", + url="/snapshots/{snapshot_name}", + path_params=path_params, + ) + + def _build_for_get_snapshot( + self, + collection_name: str, + snapshot_name: str, + ): + """ + Download specified snapshot from a collection as a file + """ + path_params = { + "collection_name": str(collection_name), + "snapshot_name": str(snapshot_name), + } + + return self.api_client.request( + type_=file, + method="GET", + url="/collections/{collection_name}/snapshots/{snapshot_name}", + path_params=path_params, + ) + + def _build_for_list_full_snapshots( + self, + ): + """ + Get list of snapshots of the whole storage + """ + return self.api_client.request( + type_=m.InlineResponse2007, + method="GET", + url="/snapshots", + ) + + def _build_for_list_snapshots( + self, + collection_name: str, + ): + """ + Get list of snapshots for a collection + """ + path_params = { + "collection_name": str(collection_name), + } + + return self.api_client.request( + type_=m.InlineResponse2007, + method="GET", + url="/collections/{collection_name}/snapshots", + path_params=path_params, + ) + + +class AsyncSnapshotsApi(_SnapshotsApi): + async def create_full_snapshot( + self, + ) -> m.InlineResponse2008: + """ + Create new snapshot of the whole storage + """ + return await self._build_for_create_full_snapshot() + + async def create_snapshot( + self, + collection_name: str, + ) -> m.InlineResponse2008: + """ + Create new snapshot for a collection + """ + return await self._build_for_create_snapshot( + collection_name=collection_name, + ) + + async def get_full_snapshot( + self, + snapshot_name: str, + ) -> file: + """ + Download specified snapshot of the whole storage as a file + """ + return await self._build_for_get_full_snapshot( + snapshot_name=snapshot_name, + ) + + async def get_snapshot( + self, + collection_name: str, + snapshot_name: str, + ) -> file: + """ + Download specified snapshot from a collection as a file + """ + return await self._build_for_get_snapshot( + collection_name=collection_name, + snapshot_name=snapshot_name, + ) + + async def list_full_snapshots( + self, + ) -> m.InlineResponse2007: + """ + Get list of snapshots of the whole storage + """ + return await self._build_for_list_full_snapshots() + + async def list_snapshots( + self, + collection_name: str, + ) -> m.InlineResponse2007: + """ + Get list of snapshots for a collection + """ + return await self._build_for_list_snapshots( + collection_name=collection_name, + ) + + +class SyncSnapshotsApi(_SnapshotsApi): + def create_full_snapshot( + self, + ) -> m.InlineResponse2008: + """ + Create new snapshot of the whole storage + """ + return self._build_for_create_full_snapshot() + + def create_snapshot( + self, + collection_name: str, + ) -> m.InlineResponse2008: + """ + Create new snapshot for a collection + """ + return self._build_for_create_snapshot( + collection_name=collection_name, + ) + + def get_full_snapshot( + self, + snapshot_name: str, + ) -> file: + """ + Download specified snapshot of the whole storage as a file + """ + return self._build_for_get_full_snapshot( + snapshot_name=snapshot_name, + ) + + def get_snapshot( + self, + collection_name: str, + snapshot_name: str, + ) -> file: + """ + Download specified snapshot from a collection as a file + """ + return self._build_for_get_snapshot( + collection_name=collection_name, + snapshot_name=snapshot_name, + ) + + def list_full_snapshots( + self, + ) -> m.InlineResponse2007: + """ + Get list of snapshots of the whole storage + """ + return self._build_for_list_full_snapshots() + + def list_snapshots( + self, + collection_name: str, + ) -> m.InlineResponse2007: + """ + Get list of snapshots for a collection + """ + return self._build_for_list_snapshots( + collection_name=collection_name, + ) diff --git a/qdrant_client/http/api_client.py b/qdrant_client/http/api_client.py index 4e6601f3..6af8ec85 100644 --- a/qdrant_client/http/api_client.py +++ b/qdrant_client/http/api_client.py @@ -6,7 +6,9 @@ from pydantic import ValidationError from qdrant_client.http.api.cluster_api import AsyncClusterApi, SyncClusterApi from qdrant_client.http.api.collections_api import AsyncCollectionsApi, SyncCollectionsApi +from qdrant_client.http.api.default_api import AsyncDefaultApi, SyncDefaultApi from qdrant_client.http.api.points_api import AsyncPointsApi, SyncPointsApi +from qdrant_client.http.api.snapshots_api import AsyncSnapshotsApi, SyncSnapshotsApi from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse ClientT = TypeVar("ClientT", bound="ApiClient") @@ -19,7 +21,9 @@ def __init__(self, host: str = None, **kwargs: Any): self.cluster_api = AsyncClusterApi(self.client) self.collections_api = AsyncCollectionsApi(self.client) + self.default_api = AsyncDefaultApi(self.client) self.points_api = AsyncPointsApi(self.client) + self.snapshots_api = AsyncSnapshotsApi(self.client) class SyncApis(Generic[ClientT]): @@ -28,7 +32,9 @@ def __init__(self, host: str = None, **kwargs: Any): self.cluster_api = SyncClusterApi(self.client) self.collections_api = SyncCollectionsApi(self.client) + self.default_api = SyncDefaultApi(self.client) self.points_api = SyncPointsApi(self.client) + self.snapshots_api = SyncSnapshotsApi(self.client) T = TypeVar("T") diff --git a/qdrant_client/http/models/models.py b/qdrant_client/http/models/models.py index 1f3332b5..0788ce0f 100644 --- a/qdrant_client/http/models/models.py +++ b/qdrant_client/http/models/models.py @@ -11,6 +11,13 @@ from pydantic.types import StrictBool, StrictInt, StrictStr +class AppBuildTelemetry(BaseModel): + version: str = Field(..., description="") + debug: bool = Field(..., description="") + web_feature: bool = Field(..., description="") + service_debug_feature: bool = Field(..., description="") + + class Batch(BaseModel): ids: List["ExtendedPointId"] = Field(..., description="") vectors: List[List[float]] = Field(..., description="") @@ -28,6 +35,13 @@ class ChangeAliasesOperation(BaseModel): ) +class ClusterConfigTelemetry(BaseModel): + enabled: bool = Field(..., description="") + grpc_timeout_ms: int = Field(..., description="") + p2p: "P2pConfigTelemetry" = Field(..., description="") + consensus: "ConsensusConfigTelemetry" = Field(..., description="") + + class ClusterStatusOneOf(BaseModel): status: Literal[ "disabled", @@ -47,6 +61,17 @@ class ClusterStatusOneOf1(BaseModel): raft_info: "RaftInfo" = Field(..., description="Description of enabled cluster") +class CollectionClusterInfo(BaseModel): + """ + Current clustering distribution for the collection + """ + + peer_id: int = Field(..., description="ID of this peer") + shard_count: int = Field(..., description="Total number of shards") + local_shards: List["LocalShardInfo"] = Field(..., description="Local shards") + remote_shards: List["RemoteShardInfo"] = Field(..., description="Remote shards") + + class CollectionConfig(BaseModel): params: "CollectionParams" = Field(..., description="") hnsw_config: "HnswConfig" = Field(..., description="") @@ -92,10 +117,28 @@ class CollectionStatus(str, Enum): RED = "red" +class CollectionTelemetry(BaseModel): + id: str = Field(..., description="") + config: "CollectionConfig" = Field(..., description="") + init_time: "Duration" = Field(..., description="") + shards: List["ShardTelemetry"] = Field(..., description="") + + class CollectionsResponse(BaseModel): collections: List["CollectionDescription"] = Field(..., description="") +class ConfigsTelemetry(BaseModel): + service_config: "ServiceConfigTelemetry" = Field(..., description="") + cluster_config: "ClusterConfigTelemetry" = Field(..., description="") + + +class ConsensusConfigTelemetry(BaseModel): + max_message_queue_size: int = Field(..., description="") + tick_period_ms: int = Field(..., description="") + bootstrap_timeout_sec: int = Field(..., description="") + + class CountRequest(BaseModel): """ Count Request Counts the number of points which satisfy the given filter. If filter is not provided, the count of all points in the collection will be returned. @@ -103,7 +146,8 @@ class CountRequest(BaseModel): filter: Optional["Filter"] = Field(None, description="Look only for points which satisfies this conditions") exact: Optional[bool] = Field( - True, description="If true, count exact number of points. If false, count approximate number of points faster." + True, + description="If true, count exact number of points. If false, count approximate number of points faster. Approximate count might be unreliable during the indexing process. Default: true", ) @@ -192,6 +236,11 @@ class Distance(str, Enum): DOT = "Dot" +class Duration(BaseModel): + secs: int = Field(..., description="") + nanos: int = Field(..., description="") + + class ErrorResponse(BaseModel): time: Optional[float] = Field(None, description="Time spent to process this request") status: Optional["ErrorResponseStatus"] = Field(None, description="") @@ -308,12 +357,42 @@ class HnswConfigDiff(BaseModel): ) +class IndexesOneOf(BaseModel): + """ + Do not use any index, scan whole vector collection during search. Guarantee 100% precision, but may be time consuming on large collections. + """ + + type: Literal["plain",] = Field( + ..., + description="Do not use any index, scan whole vector collection during search. Guarantee 100% precision, but may be time consuming on large collections.", + ) + options: Any = Field( + ..., + description="Do not use any index, scan whole vector collection during search. Guarantee 100% precision, but may be time consuming on large collections.", + ) + + +class IndexesOneOf1(BaseModel): + """ + Use filterable HNSW index for approximate search. Is very fast even on a very huge collections, but require additional space to store index and additional time to build it. + """ + + type: Literal["hnsw",] = Field( + ..., + description="Use filterable HNSW index for approximate search. Is very fast even on a very huge collections, but require additional space to store index and additional time to build it.", + ) + options: "HnswConfig" = Field( + ..., + description="Use filterable HNSW index for approximate search. Is very fast even on a very huge collections, but require additional space to store index and additional time to build it.", + ) + + class InlineResponse200(BaseModel): time: Optional[float] = Field(None, description="Time spent to process this request") status: Literal[ "ok", ] = Field(None, description="") - result: Optional["ClusterStatus"] = Field(None, description="") + result: Optional[List["TelemetryData"]] = Field(None, description="") class InlineResponse2001(BaseModel): @@ -321,7 +400,7 @@ class InlineResponse2001(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["CollectionsResponse"] = Field(None, description="") + result: Optional["ClusterStatus"] = Field(None, description="") class InlineResponse20010(BaseModel): @@ -329,10 +408,26 @@ class InlineResponse20010(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional[List["ScoredPoint"]] = Field(None, description="") + result: Optional[List["Record"]] = Field(None, description="") class InlineResponse20011(BaseModel): + time: Optional[float] = Field(None, description="Time spent to process this request") + status: Literal[ + "ok", + ] = Field(None, description="") + result: Optional["ScrollResult"] = Field(None, description="") + + +class InlineResponse20012(BaseModel): + time: Optional[float] = Field(None, description="Time spent to process this request") + status: Literal[ + "ok", + ] = Field(None, description="") + result: Optional[List["ScoredPoint"]] = Field(None, description="") + + +class InlineResponse20013(BaseModel): time: Optional[float] = Field(None, description="Time spent to process this request") status: Literal[ "ok", @@ -345,7 +440,7 @@ class InlineResponse2002(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["CollectionInfo"] = Field(None, description="") + result: Optional["CollectionsResponse"] = Field(None, description="") class InlineResponse2003(BaseModel): @@ -353,7 +448,7 @@ class InlineResponse2003(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional[bool] = Field(None, description="") + result: Optional["CollectionInfo"] = Field(None, description="") class InlineResponse2004(BaseModel): @@ -361,7 +456,7 @@ class InlineResponse2004(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["UpdateResult"] = Field(None, description="") + result: Optional[bool] = Field(None, description="") class InlineResponse2005(BaseModel): @@ -369,7 +464,7 @@ class InlineResponse2005(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional[List["SnapshotDescription"]] = Field(None, description="") + result: Optional["UpdateResult"] = Field(None, description="") class InlineResponse2006(BaseModel): @@ -377,7 +472,7 @@ class InlineResponse2006(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["SnapshotDescription"] = Field(None, description="") + result: Optional["CollectionClusterInfo"] = Field(None, description="") class InlineResponse2007(BaseModel): @@ -385,7 +480,7 @@ class InlineResponse2007(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["Record"] = Field(None, description="") + result: Optional[List["SnapshotDescription"]] = Field(None, description="") class InlineResponse2008(BaseModel): @@ -393,7 +488,7 @@ class InlineResponse2008(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional[List["Record"]] = Field(None, description="") + result: Optional["SnapshotDescription"] = Field(None, description="") class InlineResponse2009(BaseModel): @@ -401,7 +496,7 @@ class InlineResponse2009(BaseModel): status: Literal[ "ok", ] = Field(None, description="") - result: Optional["ScrollResult"] = Field(None, description="") + result: Optional["Record"] = Field(None, description="") class IsEmptyCondition(BaseModel): @@ -412,6 +507,11 @@ class IsEmptyCondition(BaseModel): is_empty: "PayloadField" = Field(..., description="Select points with empty payload for a specified field") +class LocalShardInfo(BaseModel): + shard_id: int = Field(..., description="Local shard id") + points_count: int = Field(..., description="Number of points in the shard") + + class MatchInteger(BaseModel): """ Match filter request (deprecated) @@ -502,6 +602,10 @@ class OptimizersStatusOneOf1(BaseModel): error: str = Field(..., description="Something wrong happened with optimizers") +class P2pConfigTelemetry(BaseModel): + connection_pool_size: int = Field(..., description="") + + Payload = dict @@ -536,6 +640,26 @@ class PayloadSelectorInclude(BaseModel): include: List[str] = Field(..., description="Only include this payload keys") +class PayloadStorageTypeOneOf(BaseModel): + """ + Store payload in memory and use persistence storage only if vectors are changed + """ + + type: Literal[ + "in_memory", + ] = Field(..., description="Store payload in memory and use persistence storage only if vectors are changed") + + +class PayloadStorageTypeOneOf1(BaseModel): + """ + Store payload on disk only, read each time it is requested + """ + + type: Literal[ + "on_disk", + ] = Field(..., description="Store payload on disk only, read each time it is requested") + + class PeerInfo(BaseModel): """ Information of a peer in the cluster @@ -635,6 +759,11 @@ class Record(BaseModel): vector: Optional[List[float]] = Field(None, description="Vector of the point") +class RemoteShardInfo(BaseModel): + shard_id: int = Field(..., description="Remote shard id") + peer_id: int = Field(..., description="Remote peer id") + + class RenameAlias(BaseModel): """ Change alias to a new one @@ -652,6 +781,16 @@ class RenameAliasOperation(BaseModel): rename_alias: "RenameAlias" = Field(..., description="Change alias to a new one") +class RunningEnvironmentTelemetry(BaseModel): + distribution: Optional[str] = Field(None, description="") + distribution_version: Optional[str] = Field(None, description="") + is_docker: bool = Field(..., description="") + cores: Optional[int] = Field(None, description="") + ram_size: Optional[int] = Field(None, description="") + disk_size: Optional[int] = Field(None, description="") + cpu_flags: str = Field(..., description="") + + class ScoredPoint(BaseModel): """ Search result @@ -725,11 +864,76 @@ class SearchRequest(BaseModel): ) +class SegmentConfig(BaseModel): + vector_size: int = Field(..., description="Size of a vectors used") + distance: "Distance" = Field(..., description="") + index: "Indexes" = Field(..., description="") + storage_type: "StorageType" = Field(..., description="") + payload_storage_type: Optional["PayloadStorageType"] = Field(None, description="") + + +class SegmentInfo(BaseModel): + """ + Aggregated information about segment + """ + + segment_type: "SegmentType" = Field(..., description="Aggregated information about segment") + num_vectors: int = Field(..., description="Aggregated information about segment") + num_points: int = Field(..., description="Aggregated information about segment") + num_deleted_vectors: int = Field(..., description="Aggregated information about segment") + ram_usage_bytes: int = Field(..., description="Aggregated information about segment") + disk_usage_bytes: int = Field(..., description="Aggregated information about segment") + is_appendable: bool = Field(..., description="Aggregated information about segment") + index_schema: Dict[str, "PayloadIndexInfo"] = Field(..., description="Aggregated information about segment") + + +class SegmentTelemetry(BaseModel): + info: "SegmentInfo" = Field(..., description="") + config: "SegmentConfig" = Field(..., description="") + vector_index: "VectorIndexTelemetry" = Field(..., description="") + payload_field_indices: List[Any] = Field(..., description="") + + +class SegmentType(str, Enum): + PLAIN = "plain" + INDEXED = "indexed" + SPECIAL = "special" + + +class ServiceConfigTelemetry(BaseModel): + grpc_enable: bool = Field(..., description="") + max_request_size_mb: int = Field(..., description="") + max_workers: Optional[int] = Field(None, description="") + enable_cors: bool = Field(..., description="") + + class SetPayload(BaseModel): payload: "Payload" = Field(..., description="") points: List["ExtendedPointId"] = Field(..., description="Assigns payload to each point in this list") +class ShardTelemetryOneOf(BaseModel): + remote: "ShardTelemetryOneOfRemote" = Field(..., description="") + + +class ShardTelemetryOneOf1(BaseModel): + local: "ShardTelemetryOneOf1Local" = Field(..., description="") + + +class ShardTelemetryOneOf1Local(BaseModel): + segments: List["SegmentTelemetry"] = Field(..., description="") + + +class ShardTelemetryOneOf2(BaseModel): + proxy: Any = Field(..., description="") + + +class ShardTelemetryOneOfRemote(BaseModel): + shard_id: int = Field(..., description="") + searches: "TelemetryOperationStatistics" = Field(..., description="") + updates: "TelemetryOperationStatistics" = Field(..., description="") + + class SnapshotDescription(BaseModel): name: str = Field(..., description="") creation_time: str = Field(..., description="") @@ -743,6 +947,41 @@ class StateRole(str, Enum): PRECANDIDATE = "PreCandidate" +class StorageTypeOneOf(BaseModel): + """ + Store vectors in memory and use persistence storage only if vectors are changed + """ + + type: Literal[ + "in_memory", + ] = Field(..., description="Store vectors in memory and use persistence storage only if vectors are changed") + + +class StorageTypeOneOf1(BaseModel): + """ + Use memmap to store vectors, a little slower than `InMemory`, but requires little RAM + """ + + type: Literal[ + "mmap", + ] = Field(..., description="Use memmap to store vectors, a little slower than `InMemory`, but requires little RAM") + + +class TelemetryData(BaseModel): + id: str = Field(..., description="") + app: "AppBuildTelemetry" = Field(..., description="") + system: "RunningEnvironmentTelemetry" = Field(..., description="") + configs: "ConfigsTelemetry" = Field(..., description="") + collections: List["CollectionTelemetry"] = Field(..., description="") + web: "WebApiTelemetry" = Field(..., description="") + + +class TelemetryOperationStatistics(BaseModel): + ok_count: int = Field(..., description="") + fail_count: int = Field(..., description="") + ok_avg_time: "Duration" = Field(..., description="") + + class UpdateCollection(BaseModel): """ Operation for updating parameters of the existing collection @@ -775,6 +1014,13 @@ class ValuesCount(BaseModel): lte: Optional[int] = Field(None, description="point.key.length() <= values_count.lte") +class VectorIndexTelemetry(BaseModel): + small_cardinality_searches: "TelemetryOperationStatistics" = Field(..., description="") + large_cardinality_searches: "TelemetryOperationStatistics" = Field(..., description="") + positive_check_cardinality_searches: "TelemetryOperationStatistics" = Field(..., description="") + negative_check_cardinality_searches: "TelemetryOperationStatistics" = Field(..., description="") + + class WalConfig(BaseModel): wal_capacity_mb: int = Field(..., description="Size of a single WAL segment in MB") wal_segments_ahead: int = Field(..., description="Number of WAL segments to create ahead of actually used ones") @@ -787,6 +1033,10 @@ class WalConfigDiff(BaseModel): ) +class WebApiTelemetry(BaseModel): + responses: Dict[str, int] = Field(..., description="") + + AliasOperations = Union[ CreateAliasOperation, DeleteAliasOperation, @@ -806,6 +1056,10 @@ class WalConfigDiff(BaseModel): StrictInt, StrictStr, ] +Indexes = Union[ + IndexesOneOf, + IndexesOneOf1, +] Match = Union[ MatchValue, MatchKeyword, @@ -819,6 +1073,10 @@ class WalConfigDiff(BaseModel): PayloadSelectorInclude, PayloadSelectorExclude, ] +PayloadStorageType = Union[ + PayloadStorageTypeOneOf, + PayloadStorageTypeOneOf1, +] PointInsertOperations = Union[ PointsBatch, PointsList, @@ -827,6 +1085,15 @@ class WalConfigDiff(BaseModel): PointIdsList, FilterSelector, ] +ShardTelemetry = Union[ + ShardTelemetryOneOf, + ShardTelemetryOneOf1, + ShardTelemetryOneOf2, +] +StorageType = Union[ + StorageTypeOneOf, + StorageTypeOneOf1, +] ValueVariants = Union[ StrictBool, StrictInt, diff --git a/qdrant_client/qdrant_client.py b/qdrant_client/qdrant_client.py index 113ed66f..d2ac4fd8 100644 --- a/qdrant_client/qdrant_client.py +++ b/qdrant_client/qdrant_client.py @@ -1129,3 +1129,19 @@ def create_snapshot(self, collection_name: str) -> types.SnapshotDescription: return self.openapi_client.collections_api.create_snapshot( collection_name=collection_name ).result + + def list_full_snapshots(self) -> List[types.SnapshotDescription]: + """List all snapshots for a whole storage + + Returns: + List of snapshots + """ + return self.openapi_client.snapshots_api.list_full_snapshots().result + + def create_full_snapshot(self) -> types.SnapshotDescription: + """Create snapshot for a whole storage. + + Returns: + Snapshot description + """ + return self.openapi_client.snapshots_api.create_full_snapshot().result diff --git a/tests/integration-tests.sh b/tests/integration-tests.sh index fa3123ac..f73a548a 100755 --- a/tests/integration-tests.sh +++ b/tests/integration-tests.sh @@ -11,7 +11,7 @@ function stop_docker() # Ensure current path is project root cd "$(dirname "$0")/../" -QDRANT_VERSION='v0.8.4' +QDRANT_VERSION='v0.8.5' QDRANT_HOST='localhost:6333'