[Integration][SonarQube] Added support for querying projects via the GA API

integrations/sonarqube/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- towncrier release notes start -->
 
+## 0.1.111 (2024-11-18)
+
+
+### Improvements
+
+- Increased logs presence in integration (0.1.111)
+- Replaced calls to internal API for projects to GA version, making internal API use optional (0.1.111)
+
+
 ## 0.1.110 (2024-11-12)
 
 

integrations/sonarqube/client.py

@@ -34,7 +34,8 @@ def turn_sequence_to_chunks(
 
 
 class Endpoints:
-    PROJECTS = "components/search_projects"
+    PROJECTS_INTERNAL = "components/search_projects"
+    PROJECTS = "projects/search"
     WEBHOOKS = "webhooks"
     MEASURES = "measures/component"
     BRANCHES = "project_branches/list"
@@ -95,8 +96,11 @@ async def send_api_request(
         query_params: Optional[dict[str, Any]] = None,
         json_data: Optional[dict[str, Any]] = None,
     ) -> dict[str, Any]:
+        _headers = {**self.http_client.headers, "Authorization": "Bearer <masked>"}
         logger.debug(
-            f"Sending API request to {method} {endpoint} with query params: {query_params}"
+            f"Sending {method} request to {endpoint} with headers:"
+            f" {_headers} and json data: {json_data}"
+            f" with query params: {query_params}"
         )
         try:
             response = await self.http_client.request(
@@ -125,10 +129,13 @@ async def send_paginated_api_request(
         query_params = query_params or {}
         query_params["ps"] = PAGE_SIZE
         all_resources = []  # List to hold all fetched resources
+        _headers = {**self.http_client.headers, "Authorization": "Bearer <masked>"}
 
         try:
             logger.debug(
-                f"Sending API request to {method} {endpoint} with query params: {query_params}"
+                f"Sending {method} request to {endpoint} with headers:"
+                f" {_headers} and json data: {json_data}"
+                f" with query params: {query_params}"
             )
 
             while True:
@@ -140,7 +147,12 @@ async def send_paginated_api_request(
                 )
                 response.raise_for_status()
                 response_json = response.json()
+                logger.debug(
+                    f"Received response with status code: {response.status_code}"
+                    f" and response: {response_json}"
+                )
                 resource = response_json.get(data_key, [])
+                logger.debug(f"Received {len(resource)} resources")
                 all_resources.extend(resource)
 
                 # Check for paging information and decide whether to fetch more pages
@@ -150,6 +162,7 @@ async def send_paginated_api_request(
 
                 query_params["p"] = paging_info["pageIndex"] + 1
 
+            logger.debug(f"Total resources fetched: {len(all_resources)}")
             return all_resources
 
         except httpx.HTTPStatusError as e:
@@ -175,7 +188,9 @@ async def send_paginated_api_request(
             raise
 
     async def get_components(
-        self, api_query_params: Optional[dict[str, Any]] = None
+        self,
+        endpoint: str = Endpoints.PROJECTS,
+        api_query_params: Optional[dict[str, Any]] = None,
     ) -> list[dict[str, Any]]:
         """
         Retrieve all components from SonarQube organization.
@@ -206,7 +221,7 @@ async def get_components(
 
         try:
             response = await self.send_paginated_api_request(
-                endpoint=Endpoints.PROJECTS,
+                endpoint=endpoint,
                 data_key="components",
                 query_params=query_params,
             )
@@ -288,13 +303,29 @@ async def get_all_projects(self) -> AsyncGenerator[list[dict[str, Any]], None]:
         :return (list[Any]): A list containing projects data for your organization.
         """
         logger.info(f"Fetching all projects in organization: {self.organization_id}")
-        self.metrics = cast(
-            SonarQubeProjectResourceConfig, event.resource_config
-        ).selector.metrics
-        components = await self.get_components()
-        for component in components:
-            project_data = await self.get_single_project(project=component)
-            yield [project_data]
+        selector = cast(SonarQubeProjectResourceConfig, event.resource_config).selector
+        self.metrics = selector.metrics
+
+        all_projects = {}
+        for project in await self.send_paginated_api_request(
+            endpoint=Endpoints.PROJECTS, data_key="components"
+        ):
+            project_key = project.get("key")
+            all_projects[project_key] = project
+
+        if selector.use_internal_api:
+            # usint the internal API to fetch more data
+            components = await self.get_components(Endpoints.PROJECTS_INTERNAL)
+            for component in components:
+                all_projects[component["key"]] = {
+                    **all_projects.get(component["key"], {}),
+                    **component,
+                }
+
+        yield [
+            await self.get_single_project(component)
+            for component in all_projects.values()
+        ]
 
     async def get_all_issues(self) -> AsyncGenerator[list[dict[str, Any]], None]:
         """
@@ -313,7 +344,7 @@ async def get_all_issues(self) -> AsyncGenerator[list[dict[str, Any]], None]:
         )
 
         components = await self.get_components(
-            api_query_params=project_api_query_params
+            endpoint=Endpoints.PROJECTS, api_query_params=project_api_query_params
         )
         for component in components:
             response = await self.get_issues_by_component(
@@ -366,7 +397,7 @@ async def get_all_sonarcloud_analyses(
 
         :return (list[Any]): A list containing analysis data for all components.
         """
-        components = await self.get_components()
+        components = await self.get_components(Endpoints.PROJECTS)
 
         for component in components:
             response = await self.get_analysis_by_project(component=component)
@@ -492,7 +523,7 @@ async def get_measures_for_all_pull_requests(
     async def get_all_sonarqube_analyses(
         self,
     ) -> AsyncGenerator[list[dict[str, Any]], None]:
-        components = await self.get_components()
+        components = await self.get_components(Endpoints.PROJECTS)
         for component in components:
             analysis_data = await self.get_measures_for_all_pull_requests(
                 project_key=component["key"]
@@ -575,7 +606,7 @@ async def get_or_create_webhook_url(self) -> None:
         logger.info(f"Subscribing to webhooks in organization: {self.organization_id}")
         webhook_endpoint = Endpoints.WEBHOOKS
         invoke_url = f"{self.app_host}/integration/webhook"
-        projects = await self.get_components()
+        projects = await self.get_components(Endpoints.PROJECTS)
 
         # Iterate over projects and add webhook
         webhooks_to_create = []

integrations/sonarqube/integration.py

@@ -184,6 +184,11 @@ def default_metrics() -> list[str]:
         metrics: list[str] = Field(
             description="List of metric keys", default=default_metrics()
         )
+        use_internal_api: bool = Field(
+            alias="useInternalApi",
+            description="Use internal API to fetch more data",
+            default=False,
+        )
 
     kind: Literal["projects"]
     selector: SonarQubeProjectSelector

integrations/sonarqube/main.py

@@ -25,34 +25,66 @@ def init_sonar_client() -> SonarQubeClient:
 async def on_project_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     logger.info(f"Listing Sonarqube resource: {kind}")
 
+    fetched_projects = False
+
     async for project_list in sonar_client.get_all_projects():
+        logger.info(f"Received project batch of size: {len(project_list)}")
         yield project_list
+        fetched_projects = True
+
+    if not fetched_projects:
+        logger.error("No projects found in Sonarqube")
+        raise RuntimeError(
+            "No projects found in Sonarqube, failing the resync to avoid data loss"
+        )
 
 
 @ocean.on_resync(ObjectKind.ISSUES)
 async def on_issues_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    fetched_issues = False
     async for issues_list in sonar_client.get_all_issues():
+        logger.info(f"Received issues batch of size: {len(issues_list)}")
         yield issues_list
+        fetched_issues = True
+
+    if not fetched_issues:
+        logger.error("No issues found in Sonarqube")
+        raise RuntimeError(
+            "No issues found in Sonarqube, failing the resync to avoid data loss"
+        )
 
 
 @ocean.on_resync(ObjectKind.ANALYSIS)
 @ocean.on_resync(ObjectKind.SASS_ANALYSIS)
 async def on_saas_analysis_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    fetched_analyses = False
     if not ocean.integration_config["sonar_is_on_premise"]:
+        logger.info("Sonar is not on-premise, processing SonarCloud on saas analysis")
         async for analyses_list in sonar_client.get_all_sonarcloud_analyses():
+            logger.info(f"Received analysis batch of size: {len(analyses_list)}")
             yield analyses_list
+            fetched_analyses = True
+
+        if not fetched_analyses:
+            logger.error("No analysis found in Sonarqube")
+            raise RuntimeError(
+                "No analysis found in Sonarqube, failing the resync to avoid data loss"
+            )
 
 
 @ocean.on_resync(ObjectKind.ONPREM_ANALYSIS)
 async def on_onprem_analysis_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     if ocean.integration_config["sonar_is_on_premise"]:
+        logger.info("Sonar is on-premise, processing on-premise SonarQube analysis")
         async for analyses_list in sonar_client.get_all_sonarqube_analyses():
+            logger.info(f"Received analysis batch of size: {len(analyses_list)}")
             yield analyses_list
 
 
 @ocean.on_resync(ObjectKind.PORTFOLIOS)
 async def on_portfolio_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     async for portfolio_list in sonar_client.get_all_portfolios():
+        logger.info(f"Received portfolio batch of size: {len(portfolio_list)}")
         yield portfolio_list
 
 

integrations/sonarqube/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sonarqube"
-version = "0.1.110"
+version = "0.1.111"
 description = "SonarQube projects and code quality analysis integration"
 authors = ["Port Team <[email protected]>"]