[ToolShed 2.0] Replace Infinite Scrolling GQL with Rest+Pagination

lib/galaxy/dependencies/pinned-requirements.txt

@@ -70,10 +70,6 @@ frozenlist==1.5.0
 fs==2.4.16
 fsspec==2025.2.0
 future==1.0.0
-graphene==3.4.3
-graphene-sqlalchemy==3.0.0rc2
-graphql-core==3.2.6
-graphql-relay==3.2.0
 gravity==1.0.7
 greenlet==3.1.1 ; (python_full_version < '3.14' and platform_machine == 'AMD64') or (python_full_version < '3.14' and platform_machine == 'WIN32') or (python_full_version < '3.14' and platform_machine == 'aarch64') or (python_full_version < '3.14' and platform_machine == 'amd64') or (python_full_version < '3.14' and platform_machine == 'ppc64le') or (python_full_version < '3.14' and platform_machine == 'win32') or (python_full_version < '3.14' and platform_machine == 'x86_64')
 gunicorn==23.0.0
@@ -96,6 +92,7 @@ jsonschema-specifications==2024.10.1
 kombu==5.4.2
 lagom==2.7.5
 legacy-cgi==2.6.2 ; python_full_version >= '3.13'
+limits==4.0.1
 lxml==5.3.1
 mako==1.3.9
 markdown==3.7
@@ -175,6 +172,7 @@ schema-salad==8.8.20250205075315
 setuptools==75.8.0
 setuptools-scm==5.0.2
 six==1.17.0
+slowapi==0.1.9
 sniffio==1.3.1
 social-auth-core==4.5.6
 sortedcontainers==2.4.0
@@ -184,7 +182,6 @@ sqlitedict==2.1.0
 sqlparse==0.5.3
 starlette==0.45.3
 starlette-context==0.3.6
-starlette-graphene3==0.6.0
 supervisor==4.2.5
 svgwrite==1.4.3
 tenacity==9.0.0

lib/tool_shed/managers/repositories.py

@@ -19,6 +19,8 @@
 from pydantic import BaseModel
 from sqlalchemy import (
     false,
+    func,
+    or_,
     select,
 )
 from sqlalchemy.orm import scoped_session
@@ -32,6 +34,7 @@
     ObjectNotFound,
     RequestParameterInvalidException,
 )
+from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.tool_shed.util import dependency_display
 from galaxy.util import listify
 from galaxy.util.tool_shed.encoding_util import tool_shed_encode
@@ -66,6 +69,7 @@
 from tool_shed.util.tool_util import generate_message_for_invalid_tools
 from tool_shed.webapp.model import (
     Repository,
+    RepositoryCategoryAssociation,
     RepositoryMetadata,
     User,
 )
@@ -75,10 +79,14 @@
     CreateRepositoryRequest,
     DetailedRepository,
     ExtraRepoInfo,
+    IndexSortByType,
     LegacyInstallInfoTuple,
+    PaginatedRepositoryIndexResults,
     Repository as SchemaRepository,
     RepositoryMetadataInstallInfoDict,
     ResetMetadataOnRepositoryResponse,
+    ResetMetadataOnRepositoriesRequest,
+    ResetMetadataOnRepositoriesResponse,
 )
 from .categories import get_value_mapper as category_value_mapper
 
@@ -131,10 +139,14 @@ def search(trans: ProvidesUserContext, q: str, page: int = 1, page_size: int = 1
     )
 
     results = repo_search.search(trans, search_term, page, page_size, boosts)
-    results["hostname"] = web.url_for("/", qualified=True)
+    results["hostname"] = deprecated_hostname()
     return results
 
 
+def deprecated_hostname() -> str:
+    return web.url_for("/", qualified=True)
+
+
 class UpdatesRequest(BaseModel):
     name: Optional[str] = None
     owner: Optional[str] = None
@@ -253,8 +265,41 @@ def index_tool_ids(app: ToolShedApp, tool_ids: List[str]) -> Dict[str, Any]:
         return {}
 
 
-def index_repositories(app: ToolShedApp, name: Optional[str], owner: Optional[str], deleted: bool):
-    return list(_get_repositories_by_name_and_owner_and_deleted(app.model.context, name, owner, deleted))
+class IndexRequest(BaseModel):
+    name: Optional[str] = None
+    owner: Optional[str] = None
+    deleted: bool = False
+    filter: Optional[str] = None
+    category_id: Optional[str] = None
+    sort_by: IndexSortByType = "name"
+    sort_desc: bool = False
+
+
+class PaginatedIndexRequest(IndexRequest):
+    page: int
+    page_size: int
+
+
+def index_repositories(app: ToolShedApp, index_request: IndexRequest) -> List[Repository]:
+    session = app.model.context
+    return list(session.scalars(_get_repositories_by_name_and_owner_and_deleted(app.security, index_request)))
+
+
+def index_repositories_paginated(
+    app: ToolShedApp, index_request: PaginatedIndexRequest
+) -> PaginatedRepositoryIndexResults:
+    session = app.model.context
+    stmt = _get_repositories_by_name_and_owner_and_deleted(app.security, index_request)
+    total_results = session.scalar(select(func.count()).select_from(stmt.subquery()))
+    stmt = stmt.limit(index_request.page_size).offset((index_request.page - 1) * index_request.page_size)
+    results = (to_model(app, r) for r in session.scalars(stmt).all())
+    return PaginatedRepositoryIndexResults(
+        total_results=total_results,
+        page=index_request.page,
+        page_size=index_request.page_size,
+        hits=list(results),
+        hostname=deprecated_hostname(),
+    )
 
 
 def can_manage_repo(trans: ProvidesUserContext, repository: Repository) -> bool:
@@ -456,6 +501,82 @@ def handle_repository(trans, start_time, repository):
     return ResetMetadataOnRepositoryResponse(**results)
 
 
+def reset_metadata_on_repositories(
+    trans: ProvidesUserContext, request: ResetMetadataOnRepositoriesRequest
+) -> ResetMetadataOnRepositoryResponse:
+
+    def handle_repository(trans, repository, results):
+        log.debug(f"Resetting metadata on repository {repository.name}")
+        try:
+            rmm = repository_metadata_manager.RepositoryMetadataManager(
+                trans,
+                resetting_all_metadata_on_repository=True,
+                updating_installed_repository=False,
+                repository=repository,
+                persist=False,
+            )
+            rmm.reset_all_metadata_on_repository_in_tool_shed()
+            rmm_invalid_file_tups = rmm.get_invalid_file_tups()
+            if rmm_invalid_file_tups:
+                message = generate_message_for_invalid_tools(
+                    trans.app, rmm_invalid_file_tups, repository, None, as_html=False
+                )
+                results["unsuccessful_count"] += 1
+            else:
+                message = (
+                    f"Successfully reset metadata on repository {repository.name} owned by {repository.user.username}"
+                )
+                results["successful_count"] += 1
+        except Exception as e:
+            message = (
+                f"Error resetting metadata on repository {repository.name} owned by {repository.user.username}: {e}"
+            )
+            results["unsuccessful_count"] += 1
+        status = f"{repository.name} : {message}"
+        results["repository_status"].append(status)
+        return results
+
+    start_time = strftime("%Y-%m-%d %H:%M:%S")
+    results = dict(start_time=start_time, repository_status=[], successful_count=0, unsuccessful_count=0)
+    handled_repository_ids = []
+    encoded_ids_to_skip = request.encoded_ids_to_skip or []
+    if trans.user_is_admin:
+        my_writable = request.my_writable
+    else:
+        my_writable = True
+    rmm = repository_metadata_manager.RepositoryMetadataManager(
+        trans,
+        resetting_all_metadata_on_repository=True,
+        updating_installed_repository=False,
+        persist=False,
+    )
+    # First reset metadata on all repositories of type repository_dependency_definition.
+    for repository in rmm.get_repositories_for_setting_metadata(my_writable=my_writable, order=False):
+        encoded_id = trans.security.encode_id(repository.id)
+        if encoded_id in encoded_ids_to_skip:
+            log.debug(
+                "Skipping repository with id %s because it is in encoded_ids_to_skip %s",
+                repository.id,
+                encoded_ids_to_skip,
+            )
+        elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+            results = handle_repository(trans, repository, results)
+    # Now reset metadata on all remaining repositories.
+    for repository in rmm.get_repositories_for_setting_metadata(my_writable=my_writable, order=False):
+        encoded_id = trans.security.encode_id(repository.id)
+        if encoded_id in encoded_ids_to_skip:
+            log.debug(
+                "Skipping repository with id %s because it is in encoded_ids_to_skip %s",
+                repository.id,
+                encoded_ids_to_skip,
+            )
+        elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+            results = handle_repository(trans, repository, results)
+    stop_time = strftime("%Y-%m-%d %H:%M:%S")
+    results["stop_time"] = stop_time
+    return ResetMetadataOnRepositoriesResponse(**results)
+
+
 def create_repository(trans: ProvidesUserContext, request: CreateRepositoryRequest) -> Repository:
     app: ToolShedApp = trans.app
     user = trans.user
@@ -601,14 +722,39 @@ def _get_repository_by_name_and_owner(session: scoped_session, name: str, owner:
     return session.scalars(stmt).first()
 
 
-def _get_repositories_by_name_and_owner_and_deleted(
-    session: scoped_session, name: Optional[str], owner: Optional[str], deleted: bool
-):
+def _get_repositories_by_name_and_owner_and_deleted(security: IdEncodingHelper, index_request: IndexRequest):
+    owner = index_request.owner
+    name = index_request.name
+    deleted = index_request.deleted
+    filter = index_request.filter
     stmt = select(Repository).where(Repository.deprecated == false()).where(Repository.deleted == deleted)
+    if owner is not None or filter:
+        stmt = stmt.join(Repository.user)
     if owner is not None:
         stmt = stmt.where(User.username == owner)
-        stmt = stmt.where(Repository.user_id == User.id)
     if name is not None:
         stmt = stmt.where(Repository.name == name)
-    stmt = stmt.order_by(Repository.name)
-    return session.scalars(stmt)
+    if filter:
+        filter_ilike_str = f"%{filter}%"
+        stmt = stmt.where(
+            or_(
+                User.username.ilike(filter_ilike_str),
+                Repository.name.ilike(filter_ilike_str),
+                Repository.description.ilike(filter_ilike_str),
+            )
+        )
+    if index_request.category_id is not None:
+        category_id = security.decode_id(index_request.category_id)
+        stmt = stmt.where(RepositoryCategoryAssociation.category_id == category_id)
+        stmt = stmt.where(RepositoryCategoryAssociation.repository_id == Repository.id)
+    sort_by_str = index_request.sort_by
+    sort_desc = index_request.sort_desc
+    sort_by: Any
+    if sort_by_str == "name":
+        sort_by = Repository.name
+    elif sort_by_str == "create_time":
+        sort_by = Repository.create_time
+    if sort_desc:
+        sort_by = sort_by.desc()
+    stmt = stmt.order_by(sort_by)
+    return stmt

lib/tool_shed/test/base/driver.py

@@ -76,6 +76,10 @@ def setup(self) -> None:
 
     def _setup_local(self):
         # ---- Configuration ------------------------------------------------------
+
+        # disable sensitive request limiting for the test suite
+        os.environ["TOOL_SHED_SENSITIVE_API_REQUEST_LIMIT"] = "10000/second"
+
         tool_shed_test_tmp_dir = driver_util.setup_tool_shed_tmp_dir()
         if not os.path.isdir(tool_shed_test_tmp_dir):
             os.mkdir(tool_shed_test_tmp_dir)

lib/tool_shed/test/base/populators.py

@@ -29,11 +29,13 @@
     GetOrderedInstallableRevisionsRequest,
     InstallInfo,
     OrderedInstallableRevisions,
+    PaginatedRepositoryIndexResults,
     RepositoriesByCategory,
     Repository,
     RepositoryIndexRequest,
     RepositoryIndexResponse,
     RepositoryMetadata,
+    RepositoryPaginatedIndexRequest,
     RepositorySearchRequest,
     RepositorySearchResults,
     RepositoryUpdate,
@@ -324,6 +326,15 @@ def repository_index(self, request: Optional[RepositoryIndexRequest]) -> Reposit
         api_asserts.assert_status_code_is_ok(repository_response)
         return RepositoryIndexResponse(root=repository_response.json())
 
+    def repository_index_paginated(
+        self, request: Optional[RepositoryPaginatedIndexRequest]
+    ) -> PaginatedRepositoryIndexResults:
+        repository_response = self._api_interactor.get(
+            "repositories", params=(request.model_dump() if request else {"page": 1})
+        )
+        api_asserts.assert_status_code_is_ok(repository_response)
+        return PaginatedRepositoryIndexResults(**repository_response.json())
+
     def get_usernames_allowed_to_push(self, repository: HasRepositoryId) -> List[str]:
         repository_id = self._repository_id(repository)
         show_response = self._api_interactor.get(f"repositories/{repository_id}/allow_push")