From cfc171f5b71f0982b6546b14579a72cafc71ff31 Mon Sep 17 00:00:00 2001 From: Samweli Date: Mon, 17 Apr 2023 00:28:33 +0300 Subject: [PATCH] pagination fix --- src/qgis_stac/api/models.py | 2 +- src/qgis_stac/api/network.py | 32 ++++---- src/qgis_stac/gui/qgis_stac_widget.py | 78 +++++++++---------- .../lib/pystac_client/item_search.py | 14 ++++ .../lib/pystac_client/stac_api_io.py | 41 ++++++++++ 5 files changed, 109 insertions(+), 58 deletions(-) diff --git a/src/qgis_stac/api/models.py b/src/qgis_stac/api/models.py index 1b3fe1e..f5cb03f 100644 --- a/src/qgis_stac/api/models.py +++ b/src/qgis_stac/api/models.py @@ -319,7 +319,7 @@ class Item: class ItemSearch: """ Definition for the pystac-client item search parameters""" ids: typing.Optional[list] = None - page: typing.Optional[int] = 1 + page: typing.Optional[int] = None page_size: typing.Optional[int] = 10 collections: typing.Optional[list] = None datetime: typing.Optional[QtCore.QDateTime] = None diff --git a/src/qgis_stac/api/network.py b/src/qgis_stac/api/network.py index d7dc686..e4b294d 100644 --- a/src/qgis_stac/api/network.py +++ b/src/qgis_stac/api/network.py @@ -268,25 +268,21 @@ def prepare_items_results(self, response): :rtype: list """ self.pagination = ResourcePagination() - count = 1 - items_generator = response.get_item_collections() - prev_collection = None - items_collection = None page = self.search_params.page \ - if self.search_params else Constants.PAGE_SIZE - while True: - try: - collection = next(items_generator) - prev_collection = collection - if page == count: - items_collection = collection - break - count += 1 - except StopIteration: - self.pagination.total_pages = count - items_collection = prev_collection - break - items = self.get_items_list(items_collection) + if self.search_params else None + + page_res = response.page(token=page) + + if page_res.extra_fields: + for link in page_res.extra_fields.get('links', []): + if link['rel'] == 'next': + next_page = link.get('body', {}).get('token', None) + self.pagination.next_page = next_page + if link['rel'] == 'previous': + previous_page = link.get('body', {}).get('token', None) + self.pagination.previous_page = previous_page + + items = self.get_items_list(page_res) return items def get_items_list(self, items_collection): diff --git a/src/qgis_stac/gui/qgis_stac_widget.py b/src/qgis_stac/gui/qgis_stac_widget.py index 74018c9..11dd9e3 100644 --- a/src/qgis_stac/gui/qgis_stac_widget.py +++ b/src/qgis_stac/gui/qgis_stac_widget.py @@ -163,7 +163,9 @@ def __init__( self.search_error_message = None # initialize page - self.page = 1 + self.page = None + self.previous_page = None + self.next_page = None self.total_pages = 0 self.current_collections = [] @@ -474,13 +476,12 @@ def search_items_api(self): self.current_progress_message = tr( "Searching items..." ) - self.page = 1 self.search_items() def previous_items(self): """ Sets the items search to go on the previous page. """ - self.page -= 1 + self.page = self.previous_page self.current_progress_message = tr( "Retrieving previous page..." ) @@ -489,7 +490,7 @@ def previous_items(self): def next_items(self): """ Sets the items search to go on the next page. """ - self.page += 1 + self.page = self.next_page self.current_progress_message = tr( "Retrieving next page..." ) @@ -720,49 +721,48 @@ def display_results(self, results, pagination=None): elif self.search_type == ResourceType.FEATURE: - if pagination and pagination.total_pages > 0: + if pagination: + self.previous_page = pagination.previous_page + self.next_page = pagination.next_page + + if len(results) > 0: + self.result_items_la.setText( + tr( + "Displaying page {} of results, {} item(s)" + ).format( + self.page, + len(results) + ) + ) + self.item_model = ItemsModel(results) + self.items_proxy_model.setSourceModel(self.item_model) + settings_manager.delete_all_items( + settings_manager.get_current_connection(), + self.page + ) + self.populate_results(results) + else: + + self.clear_search_results() if self.page > 1: self.page -= 1 - self.next_btn.setEnabled(False) - else: - if len(results) > 0: + if self.date_filter_group.isChecked() \ + or self.extent_box.isChecked(): self.result_items_la.setText( tr( - "Displaying page {} of results, {} item(s)" - ).format( - self.page, - len(results) + "No items were found, " + "try to expand the date filter or " + "the spatial extent filter used." ) ) - self.item_model = ItemsModel(results) - self.items_proxy_model.setSourceModel(self.item_model) - settings_manager.delete_all_items( - settings_manager.get_current_connection(), - self.page - ) - self.populate_results(results) else: - - self.clear_search_results() - if self.page > 1: - self.page -= 1 - if self.date_filter_group.isChecked() \ - or self.extent_box.isChecked(): - self.result_items_la.setText( - tr( - "No items were found, " - "try to expand the date filter or " - "the spatial extent filter used." - ) - ) - else: - self.result_items_la.setText( - tr( - "No items were found" - ) + self.result_items_la.setText( + tr( + "No items were found" ) - self.next_btn.setEnabled(len(results) > 0) - self.prev_btn.setEnabled(self.page > 1) + ) + self.next_btn.setEnabled(self.next_page is not None) + self.prev_btn.setEnabled(self.previous_page is not None) self.footprint_btn.setEnabled( False ) diff --git a/src/qgis_stac/lib/pystac_client/item_search.py b/src/qgis_stac/lib/pystac_client/item_search.py index 248b20e..29aa89a 100644 --- a/src/qgis_stac/lib/pystac_client/item_search.py +++ b/src/qgis_stac/lib/pystac_client/item_search.py @@ -435,6 +435,20 @@ def get_item_collections(self) -> Iterator[ItemCollection]: for page in self._stac_io.get_pages(self.url, self.method, self.get_parameters()): yield ItemCollection.from_dict(page, preserve_dict=False, root=self.client) + def page(self, page=None, token=None) -> Dict: + if isinstance(self._stac_io, StacApiIO): + page = self._stac_io.get_page( + self.url, + self.method, + self.get_parameters(), + page=page, + token=token, + ) + page_collection = ItemCollection.from_dict( + page, preserve_dict=False, root=self.client + ) if page is not None else None + return page_collection + def get_items(self) -> Iterator[Item]: """Iterator that yields :class:`pystac.Item` instances for each item matching the given search parameters. Calls :meth:`ItemSearch.item_collections()` internally and yields from diff --git a/src/qgis_stac/lib/pystac_client/stac_api_io.py b/src/qgis_stac/lib/pystac_client/stac_api_io.py index 1c3ff2b..7854ffc 100644 --- a/src/qgis_stac/lib/pystac_client/stac_api_io.py +++ b/src/qgis_stac/lib/pystac_client/stac_api_io.py @@ -219,6 +219,47 @@ def get_pages(self, url, method='GET', parameters={}) -> Iterator[Dict]: next_link = next((link for link in page.get('links', []) if link['rel'] == 'next'), None) + def get_page( + self, + url: str, + method: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + page: Optional = None, + token: Optional = None + ): + """Get page using the passed page number or the token + + Return: + Dict[str, Any] : JSON content from a single page + """ + if page is None and token is None: + # Always return the first page if page is not passed + parameters['page'] = 1 + page = self.read_json( + url, + method=method, + parameters=parameters + ) + if page is not None: + parameters['page'] = page + page = self.read_json( + url, + method=method, + parameters=parameters + ) + elif token is not None: + parameters['token'] = token + page = self.read_json( + url, + method=method, + parameters=parameters + ) + else: + page = None + return page + + return page + def assert_conforms_to(self, conformance_class: ConformanceClasses) -> None: """Raises a :exc:`NotImplementedError` if the API does not publish the given conformance class. This method only checks against the ``"conformsTo"`` property from the API landing page and does not make any additional