From 5f48f713d5e6579760990007b1e7cf75db778a09 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 24 Jul 2024 21:17:28 +0000 Subject: [PATCH 01/14] add support for service level --- .../newrelic/.port/resources/blueprints.json | 54 +++++++++++++++ .../.port/resources/port-app-config.yaml | 19 ++++++ integrations/newrelic/.port/spec.yaml | 1 + .../core/query_templates/service_levels.py | 63 ++++++++++++++++++ .../core/service_levels.py | 66 +++++++++++++++++++ .../newrelic/newrelic_integration/ocean.py | 9 +++ 6 files changed, 212 insertions(+) create mode 100644 integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py create mode 100644 integrations/newrelic/newrelic_integration/core/service_levels.py diff --git a/integrations/newrelic/.port/resources/blueprints.json b/integrations/newrelic/.port/resources/blueprints.json index e8bb792b46..5ac5a2fc56 100644 --- a/integrations/newrelic/.port/resources/blueprints.json +++ b/integrations/newrelic/.port/resources/blueprints.json @@ -133,5 +133,59 @@ "mirrorProperties": {}, "calculationProperties": {}, "relations": {} + }, + { + "identifier": "newRelicServiceLevel", + "description": "This blueprint represents a New Relic Service Level", + "title": "New Relic Service Level", + "icon": "NewRelic", + "schema": { + "properties": { + "description": { + "title": "Description", + "type": "string" + }, + "targetThreshold": { + "icon": "DefaultProperty", + "title": "Target Threshold", + "type": "number" + }, + "createdAt": { + "title": "Created At", + "type": "string", + "format": "date-time" + }, + "updatedAt": { + "title": "Updated At", + "type": "string", + "format": "date-time" + }, + "createdBy": { + "title": "Creator", + "type": "string", + "format": "user" + }, + "serviceLevelIndicator": { + "type": "number", + "title": "Service Level Indicator" + }, + "tags": { + "type": "object", + "title": "Tags" + } + }, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": { + "newRelicService": { + "title": "New Relic service", + "target": "newRelicService", + "required": false, + "many": false + } + } } ] diff --git a/integrations/newrelic/.port/resources/port-app-config.yaml b/integrations/newrelic/.port/resources/port-app-config.yaml index 903141f68d..c1d319d869 100644 --- a/integrations/newrelic/.port/resources/port-app-config.yaml +++ b/integrations/newrelic/.port/resources/port-app-config.yaml @@ -71,3 +71,22 @@ resources: activatedAt: .activatedAt relations: newRelicService: ".__APPLICATION.entity_guids + .__SERVICE.entity_guids" + - kind: newRelicServiceLevel + selector: + query: 'true' + port: + entity: + mappings: + blueprint: '"newRelicServiceLevel"' + identifier: .serviceLevel.indicators[0].id + title: .serviceLevel.indicators[0].name + properties: + description: .serviceLevel.indicators[0].description + targetThreshold: .serviceLevel.indicators[0].objectives[0].target + createdAt: if .serviceLevel.indicators[0].createdAt != null then (.serviceLevel.indicators[0].createdAt | tonumber / 1000 | todate) else null end + updatedAt: .serviceLevel.indicators[0].updatedAt + createdBy: .serviceLevel.indicators[0].createdBy.email + serviceLevelIndicator: .__SLI.SLI + tags: .tags + relations: + newRelicService: .serviceLevel.indicators[0].guid diff --git a/integrations/newrelic/.port/spec.yaml b/integrations/newrelic/.port/spec.yaml index 00c33d3778..372729a532 100644 --- a/integrations/newrelic/.port/spec.yaml +++ b/integrations/newrelic/.port/spec.yaml @@ -7,6 +7,7 @@ features: resources: - kind: newRelicService - kind: newRelicAlert + - kind: newRelicServiceLevel configurations: - name: newRelicAPIKey required: true diff --git a/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py b/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py new file mode 100644 index 0000000000..e80be45a3d --- /dev/null +++ b/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py @@ -0,0 +1,63 @@ +GET_SLI_BY_NRQL_QUERY = """ +{ + actor { + account(id: {{ account_id }}) { + nrql(query: "{{ nrql_query }}") { + results + } + } + } +} +""" + +LIST_SLOS_QUERY = """ +{ + actor { + entitySearch(query: "type ='SERVICE_LEVEL'") { + count + query + results{{ next_cursor_request }} { + entities { + serviceLevel { + indicators { + resultQueries { + indicator { + nrql + } + } + id + name + description + createdBy { + email + } + guid + updatedAt + createdAt + updatedBy { + email + } + objectives { + description + target + name + timeWindow { + rolling { + count + unit + } + } + } + } + } + tags { + key + values + } + } + nextCursor + } + } + } +} +""" \ No newline at end of file diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py new file mode 100644 index 0000000000..1306310b9d --- /dev/null +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -0,0 +1,66 @@ +from typing import Any, AsyncIterable, Tuple, Optional +import httpx +from port_ocean.context.ocean import ocean +from newrelic_integration.core.query_templates.service_levels import (LIST_SLOS_QUERY, GET_SLI_BY_NRQL_QUERY +) +from newrelic_integration.core.utils import send_graph_api_request +from newrelic_integration.utils import ( + render_query, +) +from newrelic_integration.core.paging import send_paginated_graph_api_request + +SLI_OBJECT = "__SLI" + +class ServiceLevelsHandler: + def __init__(self, http_client: httpx.AsyncClient): + self.http_client = http_client + + async def get_service_level_indicator_value( + self, http_client: httpx.AsyncClient, nrql: str + ) -> dict[Any, Any]: + query = await render_query( + GET_SLI_BY_NRQL_QUERY, nrql_query=nrql, account_id=ocean.integration_config.get("new_relic_account_id") + ) + response = await send_graph_api_request( + http_client, + query, + request_type="get_service_level_indicator_value" + ) + service_levels = response.get("data", {}).get("actor", {}).get("account", {}).get("nrql", {}).get("results", []) + if service_levels: + return service_levels[0] + return {} + + + async def list_service_levels( + self + ) -> AsyncIterable[dict[str, Any]]: + + async for service_level in send_paginated_graph_api_request( + self.http_client, + LIST_SLOS_QUERY, + request_type="list_service_levels", + extract_data=self._extract_service_levels + ): + nrql = service_level.get("serviceLevel", {}).get("indicators", [])[0].get("resultQueries", {}).get("indicator", {}).get("nrql") + service_level[SLI_OBJECT] = await self.get_service_level_indicator_value(self.http_client, nrql) + self._format_tags(service_level) + yield service_level + + @staticmethod + async def _extract_service_levels( + response: dict[Any, Any] + ) -> Tuple[Optional[str], list[dict[Any, Any]]]: + """Extract service levels from the response. used by send_paginated_graph_api_request""" + results = ( + response.get("data", {}) + .get("actor", {}) + .get("entitySearch", {}) + .get("results", {}) + ) + return results.get("nextCursor"), results.get("entities", []) + + @staticmethod + def _format_tags(entity: dict[Any, Any]) -> dict[Any, Any]: + entity["tags"] = {tag["key"]: tag["values"] for tag in entity.get("tags", [])} + return entity \ No newline at end of file diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index f2488acd2f..aa7e3e28fe 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -5,6 +5,8 @@ from newrelic_integration.core.entities import EntitiesHandler from newrelic_integration.core.issues import IssuesHandler, IssueState, IssueEvent +from newrelic_integration.core.service_levels import ServiceLevelsHandler + from newrelic_integration.utils import ( get_port_resource_configuration_by_newrelic_entity_type, get_port_resource_configuration_by_port_kind, @@ -62,6 +64,13 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async with httpx.AsyncClient() as http_client: yield await IssuesHandler(http_client).list_issues() +@ocean.on_resync(kind="newRelicServiceLevel") +async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + with logger.contextualize(resource_kind=kind): + async with httpx.AsyncClient() as http_client: + handler = ServiceLevelsHandler(http_client) + async for service_levels in handler.list_service_levels(): + yield [service_levels] @ocean.router.post("/events") async def handle_issues_events(issue: IssueEvent) -> dict[str, bool]: From a897f21fbb110e13e8810e102a90caa96a4e38ed Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 24 Jul 2024 21:23:31 +0000 Subject: [PATCH 02/14] create changelog --- integrations/newrelic/changelog/0.1.62.improvement.md | 1 + integrations/newrelic/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 integrations/newrelic/changelog/0.1.62.improvement.md diff --git a/integrations/newrelic/changelog/0.1.62.improvement.md b/integrations/newrelic/changelog/0.1.62.improvement.md new file mode 100644 index 0000000000..0464b95a40 --- /dev/null +++ b/integrations/newrelic/changelog/0.1.62.improvement.md @@ -0,0 +1 @@ +Added support for service level indicators and objectives \ No newline at end of file diff --git a/integrations/newrelic/pyproject.toml b/integrations/newrelic/pyproject.toml index ed49cc3292..cb85342400 100644 --- a/integrations/newrelic/pyproject.toml +++ b/integrations/newrelic/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "newrelic" -version = "0.1.61" +version = "0.1.62" description = "New Relic Integration" authors = ["Tom Tankilevitch "] From ee63e7cdaeb045d37ba237a6e4712faa496d5e32 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 24 Jul 2024 21:24:03 +0000 Subject: [PATCH 03/14] build changelog --- integrations/newrelic/CHANGELOG.md | 7 +++++++ integrations/newrelic/changelog/0.1.62.improvement.md | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) delete mode 100644 integrations/newrelic/changelog/0.1.62.improvement.md diff --git a/integrations/newrelic/CHANGELOG.md b/integrations/newrelic/CHANGELOG.md index 10e04d8f1d..492d664834 100644 --- a/integrations/newrelic/CHANGELOG.md +++ b/integrations/newrelic/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +# Port_Ocean 0.1.62 (2024-07-24) + +### Improvements + +- Added support for service level indicators and objectives + + # Port_Ocean 0.1.61 (2024-07-24) ### Improvements diff --git a/integrations/newrelic/changelog/0.1.62.improvement.md b/integrations/newrelic/changelog/0.1.62.improvement.md deleted file mode 100644 index 0464b95a40..0000000000 --- a/integrations/newrelic/changelog/0.1.62.improvement.md +++ /dev/null @@ -1 +0,0 @@ -Added support for service level indicators and objectives \ No newline at end of file From 2dcf84ba96e37399070f04827ea24bf64e6770b4 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 24 Jul 2024 21:25:20 +0000 Subject: [PATCH 04/14] lint code --- .../core/query_templates/service_levels.py | 2 +- .../core/service_levels.py | 44 ++++++++++++------- .../newrelic/newrelic_integration/ocean.py | 2 + 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py b/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py index e80be45a3d..59a6c22ca6 100644 --- a/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/query_templates/service_levels.py @@ -60,4 +60,4 @@ } } } -""" \ No newline at end of file +""" diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 1306310b9d..4ddf8bb4b9 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -1,7 +1,9 @@ from typing import Any, AsyncIterable, Tuple, Optional import httpx from port_ocean.context.ocean import ocean -from newrelic_integration.core.query_templates.service_levels import (LIST_SLOS_QUERY, GET_SLI_BY_NRQL_QUERY +from newrelic_integration.core.query_templates.service_levels import ( + LIST_SLOS_QUERY, + GET_SLI_BY_NRQL_QUERY, ) from newrelic_integration.core.utils import send_graph_api_request from newrelic_integration.utils import ( @@ -11,6 +13,7 @@ SLI_OBJECT = "__SLI" + class ServiceLevelsHandler: def __init__(self, http_client: httpx.AsyncClient): self.http_client = http_client @@ -19,31 +22,42 @@ async def get_service_level_indicator_value( self, http_client: httpx.AsyncClient, nrql: str ) -> dict[Any, Any]: query = await render_query( - GET_SLI_BY_NRQL_QUERY, nrql_query=nrql, account_id=ocean.integration_config.get("new_relic_account_id") + GET_SLI_BY_NRQL_QUERY, + nrql_query=nrql, + account_id=ocean.integration_config.get("new_relic_account_id"), ) response = await send_graph_api_request( - http_client, - query, - request_type="get_service_level_indicator_value" + http_client, query, request_type="get_service_level_indicator_value" + ) + service_levels = ( + response.get("data", {}) + .get("actor", {}) + .get("account", {}) + .get("nrql", {}) + .get("results", []) ) - service_levels = response.get("data", {}).get("actor", {}).get("account", {}).get("nrql", {}).get("results", []) if service_levels: return service_levels[0] return {} - - async def list_service_levels( - self - ) -> AsyncIterable[dict[str, Any]]: + async def list_service_levels(self) -> AsyncIterable[dict[str, Any]]: async for service_level in send_paginated_graph_api_request( self.http_client, LIST_SLOS_QUERY, request_type="list_service_levels", - extract_data=self._extract_service_levels + extract_data=self._extract_service_levels, ): - nrql = service_level.get("serviceLevel", {}).get("indicators", [])[0].get("resultQueries", {}).get("indicator", {}).get("nrql") - service_level[SLI_OBJECT] = await self.get_service_level_indicator_value(self.http_client, nrql) + nrql = ( + service_level.get("serviceLevel", {}) + .get("indicators", [])[0] + .get("resultQueries", {}) + .get("indicator", {}) + .get("nrql") + ) + service_level[SLI_OBJECT] = await self.get_service_level_indicator_value( + self.http_client, nrql + ) self._format_tags(service_level) yield service_level @@ -59,8 +73,8 @@ async def _extract_service_levels( .get("results", {}) ) return results.get("nextCursor"), results.get("entities", []) - + @staticmethod def _format_tags(entity: dict[Any, Any]) -> dict[Any, Any]: entity["tags"] = {tag["key"]: tag["values"] for tag in entity.get("tags", [])} - return entity \ No newline at end of file + return entity diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index aa7e3e28fe..c3ea24e3e6 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -64,6 +64,7 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async with httpx.AsyncClient() as http_client: yield await IssuesHandler(http_client).list_issues() + @ocean.on_resync(kind="newRelicServiceLevel") async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: with logger.contextualize(resource_kind=kind): @@ -72,6 +73,7 @@ async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async for service_levels in handler.list_service_levels(): yield [service_levels] + @ocean.router.post("/events") async def handle_issues_events(issue: IssueEvent) -> dict[str, bool]: with logger.contextualize(issue_id=issue.id, issue_state=issue.state): From dd7214643447c85491975f0fbd6e43a4c46b3d31 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Thu, 25 Jul 2024 08:17:31 +0000 Subject: [PATCH 05/14] refactor code --- integrations/newrelic/newrelic_integration/ocean.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index c3ea24e3e6..642aa14e8f 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -69,8 +69,7 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: with logger.contextualize(resource_kind=kind): async with httpx.AsyncClient() as http_client: - handler = ServiceLevelsHandler(http_client) - async for service_levels in handler.list_service_levels(): + async for service_levels in ServiceLevelsHandler(http_client).list_service_levels(): yield [service_levels] From b1ab418fb472e53a7f2f66810343b1fbe4162ae4 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Thu, 25 Jul 2024 11:21:03 +0000 Subject: [PATCH 06/14] lint code --- integrations/newrelic/newrelic_integration/ocean.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index 642aa14e8f..d694723a68 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -69,7 +69,9 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: with logger.contextualize(resource_kind=kind): async with httpx.AsyncClient() as http_client: - async for service_levels in ServiceLevelsHandler(http_client).list_service_levels(): + async for service_levels in ServiceLevelsHandler( + http_client + ).list_service_levels(): yield [service_levels] From fcc32ffdbcd9fd00e76ae9f7f8d51d5a6dbd76cd Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Thu, 25 Jul 2024 19:25:36 +0000 Subject: [PATCH 07/14] update blueprint values --- integrations/newrelic/.port/resources/blueprints.json | 4 ++-- integrations/newrelic/.port/resources/port-app-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/newrelic/.port/resources/blueprints.json b/integrations/newrelic/.port/resources/blueprints.json index 5ac5a2fc56..3a5e9edbc1 100644 --- a/integrations/newrelic/.port/resources/blueprints.json +++ b/integrations/newrelic/.port/resources/blueprints.json @@ -165,9 +165,9 @@ "type": "string", "format": "user" }, - "serviceLevelIndicator": { + "sli": { "type": "number", - "title": "Service Level Indicator" + "title": "SLI" }, "tags": { "type": "object", diff --git a/integrations/newrelic/.port/resources/port-app-config.yaml b/integrations/newrelic/.port/resources/port-app-config.yaml index c1d319d869..390b644cb4 100644 --- a/integrations/newrelic/.port/resources/port-app-config.yaml +++ b/integrations/newrelic/.port/resources/port-app-config.yaml @@ -86,7 +86,7 @@ resources: createdAt: if .serviceLevel.indicators[0].createdAt != null then (.serviceLevel.indicators[0].createdAt | tonumber / 1000 | todate) else null end updatedAt: .serviceLevel.indicators[0].updatedAt createdBy: .serviceLevel.indicators[0].createdBy.email - serviceLevelIndicator: .__SLI.SLI + sli: .__SLI.SLI tags: .tags relations: newRelicService: .serviceLevel.indicators[0].guid From ed04b60aa12bc9732eb3713d66fafcf412247214 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 30 Jul 2024 18:26:22 +0000 Subject: [PATCH 08/14] batch request sli --- .../core/service_levels.py | 28 +++++++++++++++---- .../newrelic/newrelic_integration/ocean.py | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 4ddf8bb4b9..49267685fa 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -1,5 +1,6 @@ from typing import Any, AsyncIterable, Tuple, Optional import httpx +import asyncio from port_ocean.context.ocean import ocean from newrelic_integration.core.query_templates.service_levels import ( LIST_SLOS_QUERY, @@ -12,6 +13,7 @@ from newrelic_integration.core.paging import send_paginated_graph_api_request SLI_OBJECT = "__SLI" +BATCH_SIZE = 50 class ServiceLevelsHandler: @@ -40,13 +42,23 @@ async def get_service_level_indicator_value( return service_levels[0] return {} - async def list_service_levels(self) -> AsyncIterable[dict[str, Any]]: + async def _process_service_level( + self, service_level: dict[str, Any], nrql: str + ) -> dict[str, Any]: + service_level[SLI_OBJECT] = await self.get_service_level_indicator_value( + self.http_client, nrql + ) + self._format_tags(service_level) + return service_level + async def list_service_levels(self) -> AsyncIterable[list[dict[str, Any]]]: + batch = [] async for service_level in send_paginated_graph_api_request( self.http_client, LIST_SLOS_QUERY, request_type="list_service_levels", extract_data=self._extract_service_levels, + return_batch=True, ): nrql = ( service_level.get("serviceLevel", {}) @@ -55,11 +67,15 @@ async def list_service_levels(self) -> AsyncIterable[dict[str, Any]]: .get("indicator", {}) .get("nrql") ) - service_level[SLI_OBJECT] = await self.get_service_level_indicator_value( - self.http_client, nrql - ) - self._format_tags(service_level) - yield service_level + batch.append(self._process_service_level(service_level, nrql)) + + if len(batch) >= BATCH_SIZE: + yield await asyncio.gather(*batch) + batch = [] # Clear the batch for the next set of items + + # Process any remaining items in the batch + if batch: + yield await asyncio.gather(*batch) @staticmethod async def _extract_service_levels( diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index d694723a68..0114d91f19 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -72,7 +72,7 @@ async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async for service_levels in ServiceLevelsHandler( http_client ).list_service_levels(): - yield [service_levels] + yield service_levels @ocean.router.post("/events") From 418edbe92668d08c7fb0570972f6bd23c0135537 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 30 Jul 2024 19:20:08 +0000 Subject: [PATCH 09/14] use stream async iterator --- .../core/service_levels.py | 33 +++++++------------ .../newrelic/newrelic_integration/ocean.py | 28 +++++++++++++--- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 49267685fa..72adcd6004 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -13,7 +13,6 @@ from newrelic_integration.core.paging import send_paginated_graph_api_request SLI_OBJECT = "__SLI" -BATCH_SIZE = 50 class ServiceLevelsHandler: @@ -42,40 +41,30 @@ async def get_service_level_indicator_value( return service_levels[0] return {} - async def _process_service_level( - self, service_level: dict[str, Any], nrql: str + async def process_service_level( + self, service_level: dict[str, Any] ) -> dict[str, Any]: + nrql = ( + service_level.get("serviceLevel", {}) + .get("indicators", [])[0] + .get("resultQueries", {}) + .get("indicator", {}) + .get("nrql") + ) service_level[SLI_OBJECT] = await self.get_service_level_indicator_value( self.http_client, nrql ) self._format_tags(service_level) return service_level - async def list_service_levels(self) -> AsyncIterable[list[dict[str, Any]]]: - batch = [] + async def list_service_levels(self) -> AsyncIterable[dict[str, Any]]: async for service_level in send_paginated_graph_api_request( self.http_client, LIST_SLOS_QUERY, request_type="list_service_levels", extract_data=self._extract_service_levels, - return_batch=True, ): - nrql = ( - service_level.get("serviceLevel", {}) - .get("indicators", [])[0] - .get("resultQueries", {}) - .get("indicator", {}) - .get("nrql") - ) - batch.append(self._process_service_level(service_level, nrql)) - - if len(batch) >= BATCH_SIZE: - yield await asyncio.gather(*batch) - batch = [] # Clear the batch for the next set of items - - # Process any remaining items in the batch - if batch: - yield await asyncio.gather(*batch) + yield service_level @staticmethod async def _extract_service_levels( diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index 0114d91f19..6b2a3dacc1 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -1,7 +1,9 @@ +from typing import Any, AsyncIterable import httpx from loguru import logger from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE +from port_ocean.utils.async_iterators import stream_async_iterators_tasks from newrelic_integration.core.entities import EntitiesHandler from newrelic_integration.core.issues import IssuesHandler, IssueState, IssueEvent @@ -12,6 +14,13 @@ get_port_resource_configuration_by_port_kind, ) +BATCH_SIZE = 20 + + +## stream_async_iterators_tasks expects a list of async iterators, so we need to wrap the coroutine in a list +async def wrap_coroutine(coroutine: Any) -> AsyncIterable[Any]: + yield await coroutine + @ocean.on_resync() async def resync_entities(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: @@ -69,10 +78,21 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: with logger.contextualize(resource_kind=kind): async with httpx.AsyncClient() as http_client: - async for service_levels in ServiceLevelsHandler( - http_client - ).list_service_levels(): - yield service_levels + handler = ServiceLevelsHandler(http_client) + batch = [] + async for service_level in handler.list_service_levels(): + batch.append( + wrap_coroutine(handler.process_service_level(service_level)) + ) + if len(batch) >= BATCH_SIZE: + async for item in stream_async_iterators_tasks(*batch): + yield [item] + batch = [] + + # Process any remaining items in the batch + if batch: + async for item in stream_async_iterators_tasks(*batch): + yield [item] @ocean.router.post("/events") From 7982ea554d62933a483468559fa41cb25d2c54a2 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 30 Jul 2024 19:24:07 +0000 Subject: [PATCH 10/14] remove unused import --- .../newrelic/newrelic_integration/core/service_levels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 72adcd6004..870ad06801 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -1,6 +1,5 @@ from typing import Any, AsyncIterable, Tuple, Optional import httpx -import asyncio from port_ocean.context.ocean import ocean from newrelic_integration.core.query_templates.service_levels import ( LIST_SLOS_QUERY, From ff43d1cd5501208f29c065af44730653d799f76a Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 30 Jul 2024 19:36:10 +0000 Subject: [PATCH 11/14] add comment --- .../newrelic/newrelic_integration/core/service_levels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 870ad06801..173cc817b2 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -43,6 +43,7 @@ async def get_service_level_indicator_value( async def process_service_level( self, service_level: dict[str, Any] ) -> dict[str, Any]: + # Get the NRQL which is used to build the actual SLI result nrql = ( service_level.get("serviceLevel", {}) .get("indicators", [])[0] From 926cf6c6478ea39de628c3c2833c6758dc2a7072 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Fri, 2 Aug 2024 13:12:00 +0000 Subject: [PATCH 12/14] add semaphoes --- .../core/service_levels.py | 16 +++++-- .../newrelic/newrelic_integration/ocean.py | 42 +++++++++---------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index 173cc817b2..f1a92776f3 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -12,6 +12,7 @@ from newrelic_integration.core.paging import send_paginated_graph_api_request SLI_OBJECT = "__SLI" +BATCH_SIZE = 50 class ServiceLevelsHandler: @@ -40,7 +41,7 @@ async def get_service_level_indicator_value( return service_levels[0] return {} - async def process_service_level( + async def enrich_slo_with_sli_and_tags( self, service_level: dict[str, Any] ) -> dict[str, Any]: # Get the NRQL which is used to build the actual SLI result @@ -57,14 +58,23 @@ async def process_service_level( self._format_tags(service_level) return service_level - async def list_service_levels(self) -> AsyncIterable[dict[str, Any]]: + async def list_service_levels(self) -> AsyncIterable[list[dict[str, Any]]]: + batch = [] async for service_level in send_paginated_graph_api_request( self.http_client, LIST_SLOS_QUERY, request_type="list_service_levels", extract_data=self._extract_service_levels, ): - yield service_level + batch.append(service_level) + + if len(batch) >= BATCH_SIZE: + yield batch + batch = [] # Clear the batch for the next set of items + + # Process any remaining items in the batch + if batch: + yield batch @staticmethod async def _extract_service_levels( diff --git a/integrations/newrelic/newrelic_integration/ocean.py b/integrations/newrelic/newrelic_integration/ocean.py index 6b2a3dacc1..5f36b96252 100644 --- a/integrations/newrelic/newrelic_integration/ocean.py +++ b/integrations/newrelic/newrelic_integration/ocean.py @@ -1,10 +1,9 @@ -from typing import Any, AsyncIterable +from typing import Any import httpx +import asyncio from loguru import logger from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE -from port_ocean.utils.async_iterators import stream_async_iterators_tasks - from newrelic_integration.core.entities import EntitiesHandler from newrelic_integration.core.issues import IssuesHandler, IssueState, IssueEvent from newrelic_integration.core.service_levels import ServiceLevelsHandler @@ -14,12 +13,16 @@ get_port_resource_configuration_by_port_kind, ) -BATCH_SIZE = 20 +MAX_CONCURRENT_REQUESTS = 10 -## stream_async_iterators_tasks expects a list of async iterators, so we need to wrap the coroutine in a list -async def wrap_coroutine(coroutine: Any) -> AsyncIterable[Any]: - yield await coroutine +async def enrich_service_level( + handler: ServiceLevelsHandler, + semaphore: asyncio.Semaphore, + service_level: dict[str, Any], +) -> dict[str, Any]: + async with semaphore: + return await handler.enrich_slo_with_sli_and_tags(service_level) @ocean.on_resync() @@ -78,21 +81,18 @@ async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: async def resync_service_levels(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: with logger.contextualize(resource_kind=kind): async with httpx.AsyncClient() as http_client: - handler = ServiceLevelsHandler(http_client) - batch = [] - async for service_level in handler.list_service_levels(): - batch.append( - wrap_coroutine(handler.process_service_level(service_level)) - ) - if len(batch) >= BATCH_SIZE: - async for item in stream_async_iterators_tasks(*batch): - yield [item] - batch = [] + service_level_handler = ServiceLevelsHandler(http_client) + semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS) - # Process any remaining items in the batch - if batch: - async for item in stream_async_iterators_tasks(*batch): - yield [item] + async for service_levels in service_level_handler.list_service_levels(): + tasks = [ + enrich_service_level( + service_level_handler, semaphore, service_level + ) + for service_level in service_levels + ] + enriched_service_levels = await asyncio.gather(*tasks) + yield enriched_service_levels @ocean.router.post("/events") From cd57ac685a9f2b59c0ecda6dc4b148395e22373b Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Fri, 2 Aug 2024 13:19:53 +0000 Subject: [PATCH 13/14] update comment --- .../newrelic/newrelic_integration/core/service_levels.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrations/newrelic/newrelic_integration/core/service_levels.py b/integrations/newrelic/newrelic_integration/core/service_levels.py index f1a92776f3..5966e1e51e 100644 --- a/integrations/newrelic/newrelic_integration/core/service_levels.py +++ b/integrations/newrelic/newrelic_integration/core/service_levels.py @@ -70,9 +70,8 @@ async def list_service_levels(self) -> AsyncIterable[list[dict[str, Any]]]: if len(batch) >= BATCH_SIZE: yield batch - batch = [] # Clear the batch for the next set of items + batch = [] # Clearing the batch for the next set of items - # Process any remaining items in the batch if batch: yield batch From 20650277339f708c4b3e5c30542c12377cc699b4 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Thu, 15 Aug 2024 11:21:57 +0000 Subject: [PATCH 14/14] update changelog formatting --- integrations/newrelic/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/newrelic/CHANGELOG.md b/integrations/newrelic/CHANGELOG.md index afd6314f37..467f03dbb2 100644 --- a/integrations/newrelic/CHANGELOG.md +++ b/integrations/newrelic/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 -# Port_Ocean 0.1.70 (2024-08-15) +## 0.1.70 (2024-08-15) ### Improvements