[Integration][Sonarqube] Bug Fix send_paginated_request pagination stops prematurely

integrations/sonarqube/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- towncrier release notes start -->
 
+## 0.1.121 (2024-12-19)
+
+
+### Bug Fixes
+
+- Fixes bug in pagination, pagination to use total record count instead of response size, preventing early termination
+
+
 ## 0.1.120 (2024-12-15)
 
 

integrations/sonarqube/client.py

@@ -121,7 +121,6 @@ async def send_paginated_api_request(
         query_params: Optional[dict[str, Any]] = None,
         json_data: Optional[dict[str, Any]] = None,
     ) -> list[dict[str, Any]]:
-
         query_params = query_params or {}
         query_params["ps"] = PAGE_SIZE
         all_resources = []  # List to hold all fetched resources
@@ -146,10 +145,20 @@ async def send_paginated_api_request(
 
                 # Check for paging information and decide whether to fetch more pages
                 paging_info = response_json.get("paging")
-                if paging_info is None or len(resource) < PAGE_SIZE:
+                if not paging_info:
+                    break
+
+                page_index = paging_info.get(
+                    "pageIndex", 1
+                )  # SonarQube pageIndex starts at 1
+                page_size = paging_info.get("pageSize", PAGE_SIZE)
+                total_records = paging_info.get("total", 0)
+
+                # Check if we have fetched all records
+                if page_index * page_size >= total_records:
                     break
 
-                query_params["p"] = paging_info["pageIndex"] + 1
+                query_params["p"] = page_index + 1
 
             return all_resources
 

integrations/sonarqube/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sonarqube"
-version = "0.1.120"
+version = "0.1.121"
 description = "SonarQube projects and code quality analysis integration"
 authors = ["Port Team <[email protected]>"]
 

integrations/sonarqube/tests/test_client.py

@@ -1,8 +1,28 @@
 from typing import Any
-
 import pytest
+from unittest.mock import AsyncMock, patch, MagicMock
+import httpx
+from client import SonarQubeClient, turn_sequence_to_chunks
+
+from port_ocean.context.ocean import initialize_port_ocean_context
+from port_ocean.exceptions.context import PortOceanContextAlreadyInitializedError
+
 
-from client import turn_sequence_to_chunks
+@pytest.fixture(autouse=True)
+def mock_ocean_context() -> None:
+    """Fixture to mock the Ocean context initialization."""
+    try:
+        mock_ocean_app = MagicMock()
+        mock_ocean_app.config.integration.config = {
+            "sonarqube_host": "https://example.sonarqube.com",
+            "sonarqube_token": "test_token",
+            "organization_id": "test_org",
+        }
+        mock_ocean_app.integration_router = MagicMock()
+        mock_ocean_app.port_client = MagicMock()
+        initialize_port_ocean_context(mock_ocean_app)
+    except PortOceanContextAlreadyInitializedError:
+        pass
 
 
 @pytest.mark.parametrize(
@@ -18,3 +38,63 @@ def test_turn_sequence_to_chunks(
     input: list[Any], output: list[list[Any]], chunk_size: int
 ) -> None:
     assert list(turn_sequence_to_chunks(input, chunk_size)) == output
+
+
+@pytest.mark.asyncio
+async def test_pagination() -> None:
+    client = SonarQubeClient("http://test", "token", None, None)
+
+    mock_responses = [
+        httpx.Response(
+            200,
+            json={
+                "components": [{"id": "1"}],
+                "paging": {"pageIndex": 1, "pageSize": 1, "total": 2},
+            },
+            request=httpx.Request("GET", "/"),
+        ),
+        httpx.Response(
+            200,
+            json={
+                "components": [{"id": "2"}],
+                "paging": {"pageIndex": 2, "pageSize": 1, "total": 2},
+            },
+            request=httpx.Request("GET", "/"),
+        ),
+    ]
+
+    with patch.object(
+        client.http_client, "request", AsyncMock(side_effect=mock_responses)
+    ):
+        result = await client.send_paginated_api_request("test", "components")
+        assert len(result) == 2
+
+
+@pytest.mark.asyncio
+async def test_pagination_partial_response() -> None:
+    client = SonarQubeClient("http://test", "token", None, None)
+
+    mock_responses = [
+        httpx.Response(
+            200,
+            json={
+                "components": [{"id": i} for i in range(3)],
+                "paging": {"pageIndex": 1, "pageSize": 3, "total": 4},
+            },
+            request=httpx.Request("GET", "/"),
+        ),
+        httpx.Response(
+            200,
+            json={
+                "components": [{"id": 3}],
+                "paging": {"pageIndex": 2, "pageSize": 3, "total": 4},
+            },
+            request=httpx.Request("GET", "/"),
+        ),
+    ]
+
+    with patch.object(
+        client.http_client, "request", AsyncMock(side_effect=mock_responses)
+    ):
+        result = await client.send_paginated_api_request("test", "components")
+        assert len(result) == 4