Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

misc: Make page token required #813

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions encord/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
upload_to_signed_url_list,
)
from encord.http.v2.api_client import ApiClient
from encord.http.v2.payloads import Page
from encord.http.v2.payloads import BulkResponse
from encord.orm.analytics import (
CollaboratorTimer,
CollaboratorTimerParams,
Expand Down Expand Up @@ -305,9 +305,9 @@ def add_users(self, user_emails: List[str], user_role: DatasetUserRole) -> List[

return [DatasetUser.from_dict(user) for user in users]

def list_groups(self, dataset_hash: uuid.UUID) -> Page[DatasetGroup]:
def list_groups(self, dataset_hash: uuid.UUID) -> BulkResponse[DatasetGroup]:
return self._get_api_client().get(
f"datasets/{dataset_hash}/groups", params=None, result_type=Page[DatasetGroup]
f"datasets/{dataset_hash}/groups", params=None, result_type=BulkResponse[DatasetGroup]
)

def add_groups(self, dataset_hash: str, group_hash: List[uuid.UUID], user_role: DatasetUserRole) -> None:
Expand Down Expand Up @@ -954,9 +954,9 @@ def add_users(self, user_emails: List[str], user_role: ProjectUserRole) -> List[

return [ProjectUser.from_dict(user) for user in users]

def list_groups(self, project_hash: uuid.UUID) -> Page[ProjectGroup]:
def list_groups(self, project_hash: uuid.UUID) -> BulkResponse[ProjectGroup]:
return self._get_api_client().get(
f"projects/{project_hash}/groups", params=None, result_type=Page[ProjectGroup]
f"projects/{project_hash}/groups", params=None, result_type=BulkResponse[ProjectGroup]
)

def add_groups(self, project_hash: uuid.UUID, group_hash: List[uuid.UUID], user_role: ProjectUserRole) -> None:
Expand Down Expand Up @@ -1134,7 +1134,7 @@ def remove_datasets(self, dataset_hashes: List[str]) -> bool:
def list_project_datasets(self, project_hash: UUID) -> Iterable[ProjectDataset]:
return (
self._get_api_client()
.get(f"projects/{project_hash}/datasets", params=None, result_type=Page[ProjectDataset])
.get(f"projects/{project_hash}/datasets", params=None, result_type=BulkResponse[ProjectDataset])
.results
)

Expand Down
6 changes: 5 additions & 1 deletion encord/http/v2/payloads.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,8 @@

class Page(GenericBaseDTO, Generic[T]):
results: List[T]
next_page_token: Optional[str] = None
next_page_token: Optional[str]


class BulkResponse(GenericBaseDTO, Generic[T]):
results: List[T]
6 changes: 4 additions & 2 deletions encord/ontology.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from uuid import UUID

from encord.http.v2.api_client import ApiClient
from encord.http.v2.payloads import Page
from encord.http.v2.payloads import BulkResponse
from encord.objects.ontology_structure import OntologyStructure
from encord.orm.group import AddOntologyGroupsPayload, OntologyGroup, RemoveGroupsParams
from encord.orm.ontology import CreateOrUpdateOntologyPayload
Expand Down Expand Up @@ -132,7 +132,9 @@ def list_groups(self) -> Iterable[OntologyGroup]:
List all groups that have access to a particular ontology.
"""
ontology_hash = convert_to_uuid(self.ontology_hash)
page = self.api_client.get(f"ontologies/{ontology_hash}/groups", params=None, result_type=Page[OntologyGroup])
page = self.api_client.get(
f"ontologies/{ontology_hash}/groups", params=None, result_type=BulkResponse[OntologyGroup]
)

yield from page.results

Expand Down
14 changes: 7 additions & 7 deletions encord/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from encord.http.constants import DEFAULT_REQUESTS_SETTINGS
from encord.http.utils import CloudUploadSettings, _upload_single_file
from encord.http.v2.api_client import ApiClient
from encord.http.v2.payloads import Page
from encord.http.v2.payloads import BulkResponse
from encord.orm.dataset import LongPollingStatus
from encord.orm.group import AddStorageFolderGroupsPayload, RemoveGroupsParams, StorageFolderGroup
from encord.orm.storage import (
Expand Down Expand Up @@ -1020,7 +1020,7 @@ def list_groups(self) -> Iterable[StorageFolderGroup]:
List all groups that have access to this folder.
"""
page = self._api_client.get(
f"/storage/folders/{self.uuid}/groups", params=None, result_type=Page[StorageFolderGroup]
f"/storage/folders/{self.uuid}/groups", params=None, result_type=BulkResponse[StorageFolderGroup]
)

yield from page.results
Expand Down Expand Up @@ -1086,7 +1086,7 @@ def _get_upload_signed_urls(
count=count,
frames_subfolder_name=frames_subfolder_name,
),
result_type=Page[orm_storage.UploadSignedUrl],
result_type=BulkResponse[orm_storage.UploadSignedUrl],
)

return urls.results
Expand Down Expand Up @@ -1334,7 +1334,7 @@ def _patch_multiple_folders(
"storage/folders/patch-bulk",
params=None,
payload=orm_storage.PatchFoldersBulkPayload(folder_patches=folder_patches),
result_type=Page[orm_storage.StorageFolder],
result_type=BulkResponse[orm_storage.StorageFolder],
).results

@staticmethod
Expand Down Expand Up @@ -1589,7 +1589,7 @@ def get_child_items(self, get_signed_urls: bool = False) -> Iterable["StorageIte
child_items = self._api_client.get(
f"/storage/folders/{self.parent_folder_uuid}/items/{self.uuid}/child-items",
params=orm_storage.GetChildItemsParams(sign_urls=get_signed_urls),
result_type=Page[orm_storage.StorageItem],
result_type=BulkResponse[orm_storage.StorageItem],
).results

return [StorageItem(self._api_client, item) for item in child_items]
Expand Down Expand Up @@ -1737,7 +1737,7 @@ def _get_items(api_client: ApiClient, item_uuids: List[UUID], get_signed_url: bo
"storage/items/get-bulk",
params=None,
payload=GetItemsBulkPayload(item_uuids=item_uuids, sign_urls=get_signed_url),
result_type=Page[orm_storage.StorageItem],
result_type=BulkResponse[orm_storage.StorageItem],
)
return [StorageItem(api_client, orm_item) for orm_item in orm_items.results]

Expand All @@ -1750,7 +1750,7 @@ def _patch_multiple_items(
"storage/items/patch-bulk",
params=None,
payload=orm_storage.PatchItemsBulkPayload(item_patches=item_patches),
result_type=Page[orm_storage.StorageItem],
result_type=BulkResponse[orm_storage.StorageItem],
).results

def _set_orm_item(self, orm_item: orm_storage.StorageItem) -> None:
Expand Down
8 changes: 5 additions & 3 deletions encord/user_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
upload_to_signed_url_list,
)
from encord.http.v2.api_client import ApiClient
from encord.http.v2.payloads import Page
from encord.http.v2.payloads import BulkResponse
from encord.metadata_schema import MetadataSchema
from encord.objects import OntologyStructure
from encord.objects.common import (
Expand Down Expand Up @@ -811,7 +811,9 @@ def get_ontologies(
"""
properties_filter = OntologiesFilterParams.from_dict(self.__validate_filter(locals()))
properties_filter.include_org_access = include_org_access
page = self._api_client.get("ontologies", params=properties_filter, result_type=Page[OntologyWithUserRole])
page = self._api_client.get(
"ontologies", params=properties_filter, result_type=BulkResponse[OntologyWithUserRole]
)

# a hack to be able to share validation code without too much c&p
retval: List[Dict] = []
Expand Down Expand Up @@ -882,7 +884,7 @@ def list_groups(self) -> Iterable[OrmGroup]:
"""
List all groups belonging to the user's current organization.
"""
page = self._api_client.get("user/current-organisation/groups", params=None, result_type=Page[OrmGroup])
page = self._api_client.get("user/current-organisation/groups", params=None, result_type=BulkResponse[OrmGroup])
yield from page.results

def deidentify_dicom_files_start(
Expand Down
4 changes: 2 additions & 2 deletions tests/http/test_timeout_overrides_setting.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,8 @@ def test_request_timeout_settings_correctly_propagated(send: MagicMock, api_clie
# Re-mocking for V2 API
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"results": []}
mock_response.content = json.dumps({"results": []})
mock_response.json.return_value = {"results": [], "next_page_token": None}
mock_response.content = json.dumps({"results": [], "next_page_token": None})
send.side_effect = None
send.return_value = mock_response

Expand Down
8 changes: 5 additions & 3 deletions tests/test_analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def construct_timer(
def test_project_collaborator_timers_empty_page(api_client_get: MagicMock, project: Project):
after_time = parse_datetime("2023-01-01T21:00:00")

api_client_get.return_value = Page[CollaboratorTimer](results=[])
api_client_get.return_value = Page[CollaboratorTimer](results=[], next_page_token=None)

timers = list(project.list_collaborator_timers(after=after_time))

Expand All @@ -48,7 +48,8 @@ def test_project_collaborator_timers_single_page(api_client_get: MagicMock, proj
results=[
construct_timer(data_title="data title 1"),
construct_timer(data_title="data title 2"),
]
],
next_page_token=None,
)

api_client_get.return_value = return_value
Expand Down Expand Up @@ -86,7 +87,8 @@ def test_project_collaborator_timers_multi_page(api_client_get: MagicMock, proje
results=[
construct_timer(data_title="data title 3"),
construct_timer(data_title="data title 4"),
]
],
next_page_token=None,
)

api_client_get.side_effect = [return_value_page_1, return_value_page_2]
Expand Down
2 changes: 1 addition & 1 deletion tests/test_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def test_label_rows_property_queries_metadata(project_client_mock: MagicMock, pr
def test_project_datasets(api_get: MagicMock, project: Project) -> None:
dataset_hash = uuid.uuid4()
expected_dataset = ProjectDataset(dataset_hash=dataset_hash, title="test dataset", description="my test dataset")
api_get.return_value = Page(results=[expected_dataset])
api_get.return_value = Page(results=[expected_dataset], next_page_token=None)

assert len(list(project.list_datasets())) == 1
assert list(project.list_datasets()) == [expected_dataset]
Expand Down
6 changes: 2 additions & 4 deletions tests/test_user_client_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
from requests import PreparedRequest, Session

from encord.configs import SshConfig
from encord.http.v2.payloads import Page
from encord.orm.analytics import CollaboratorTimer
from encord.orm.project import Project as OrmProject
from encord.orm.project import ProjectDTO, ProjectType
from encord.user_client import EncordUserClient
Expand Down Expand Up @@ -108,8 +106,8 @@ def side_effects(*args, **kwargs):
elif args[0].path_url.startswith("/v2/public/analytics/collaborators/timers"):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = Page[CollaboratorTimer](results=[]).to_dict()
mock_response.json.content = json.dumps(Page[CollaboratorTimer](results=[]).to_dict())
mock_response.json.return_value = {"results": [], "next_page_token": None}
mock_response.json.content = json.dumps({"results": [], "next_page_token": None})
return mock_response
elif args[0].method == "GET" and args[0].path_url.startswith("/v2/public/ontologies/"):
mock_response = MagicMock()
Expand Down
Loading