From 551352e94103f2283920bebf3aa5f15208319a99 Mon Sep 17 00:00:00 2001 From: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com> Date: Sun, 31 Dec 2023 17:53:13 +0200 Subject: [PATCH] [Core] Add support to create pages as part of integration setup (#308) --- CHANGELOG.md | 11 +++ port_ocean/clients/port/mixins/blueprints.py | 33 +++++++- port_ocean/core/defaults/common.py | 2 + port_ocean/core/defaults/initialize.py | 77 ++++++++++++++++++- .../entities_state_applier/port/applier.py | 2 + .../core/integrations/mixins/sync_raw.py | 4 + port_ocean/exceptions/port_defaults.py | 8 +- pyproject.toml | 2 +- 8 files changed, 131 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa91bef4b2..9cfdcaae5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.4.13 (2023-12-31) + +### Features + +- Added capability to create pages as part of the integration setup (PORT-5689) + +### Improvements + +- Added integration and blueprints existence check before creating default resources (#1) +- Added verbosity to diff deletion process after resync (#2) + ## 0.4.12 (2023-12-22) diff --git a/port_ocean/clients/port/mixins/blueprints.py b/port_ocean/clients/port/mixins/blueprints.py index f3c3eeb4e6..24f9ee6453 100644 --- a/port_ocean/clients/port/mixins/blueprints.py +++ b/port_ocean/clients/port/mixins/blueprints.py @@ -14,13 +14,15 @@ def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient): self.auth = auth self.client = client - async def get_blueprint(self, identifier: str) -> Blueprint: + async def get_blueprint( + self, identifier: str, should_log: bool = True + ) -> Blueprint: logger.info(f"Fetching blueprint with id: {identifier}") response = await self.client.get( f"{self.auth.api_url}/blueprints/{identifier}", headers=await self.auth.headers(), ) - handle_status_code(response) + handle_status_code(response, should_log=should_log) return Blueprint.parse_obj(response.json()["blueprint"]) async def create_blueprint( @@ -105,3 +107,30 @@ async def create_scorecard( ) handle_status_code(response) + + async def create_page( + self, + page: dict[str, Any], + ) -> dict[str, Any]: + logger.info(f"Creating page: {page}") + response = await self.client.post( + f"{self.auth.api_url}/pages", + json=page, + headers=await self.auth.headers(), + ) + + handle_status_code(response) + return page + + async def delete_page( + self, + identifier: str, + should_raise: bool = False, + ) -> None: + logger.info(f"Deleting page: {identifier}") + response = await self.client.delete( + f"{self.auth.api_url}/pages/{identifier}", + headers=await self.auth.headers(), + ) + + handle_status_code(response, should_raise) diff --git a/port_ocean/core/defaults/common.py b/port_ocean/core/defaults/common.py index f9e208b0c5..a0ea9ba323 100644 --- a/port_ocean/core/defaults/common.py +++ b/port_ocean/core/defaults/common.py @@ -26,6 +26,7 @@ class Defaults(BaseModel): blueprints: list[dict[str, Any]] = [] actions: list[Preset] = [] scorecards: list[Preset] = [] + pages: list[dict[str, Any]] = [] port_app_config: Optional[PortAppConfig] = Field( default=None, alias="port-app-config" ) @@ -108,6 +109,7 @@ def get_port_integration_defaults( blueprints=default_jsons.get("blueprints", []), actions=default_jsons.get("actions", []), scorecards=default_jsons.get("scorecards", []), + pages=default_jsons.get("pages", []), port_app_config=port_app_config_class( **default_jsons.get("port-app-config", {}) ), diff --git a/port_ocean/core/defaults/initialize.py b/port_ocean/core/defaults/initialize.py index f541789693..d26f765694 100644 --- a/port_ocean/core/defaults/initialize.py +++ b/port_ocean/core/defaults/initialize.py @@ -2,6 +2,7 @@ from typing import Type, Any import httpx +from starlette import status from loguru import logger from port_ocean.clients.port.client import PortClient @@ -10,6 +11,7 @@ from port_ocean.context.ocean import ocean from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults from port_ocean.core.handlers.port_app_config.models import PortAppConfig +from port_ocean.core.models import Blueprint from port_ocean.exceptions.port_defaults import ( AbortDefaultCreationError, ) @@ -52,10 +54,38 @@ async def _create_resources( defaults: Defaults, integration_config: IntegrationConfiguration, ) -> None: + response = await port_client._get_current_integration() + if response.status_code == status.HTTP_404_NOT_FOUND: + logger.info("Integration doesn't exist, creating new integration") + else: + logger.info("Integration already exists, skipping integration creation...") + return + creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps( defaults.blueprints ) + blueprints_results = await asyncio.gather( + *( + port_client.get_blueprint(blueprint["identifier"], should_log=False) + for blueprint in creation_stage + ), + return_exceptions=True, + ) + + existing_blueprints = [ + result.identifier + for result in blueprints_results + if not isinstance(result, httpx.HTTPStatusError) + and isinstance(result, Blueprint) + ] + + if existing_blueprints: + logger.info( + f"Blueprints already exist: {existing_blueprints}. Skipping integration default creation..." + ) + return + create_results = await asyncio.gather( *( port_client.create_blueprint( @@ -81,7 +111,7 @@ async def _create_resources( ) raise AbortDefaultCreationError(created_blueprints, errors) - + created_pages = [] try: for patch_stage in blueprint_patches: await asyncio.gather( @@ -111,16 +141,42 @@ async def _create_resources( ) ) + create_pages_result = await asyncio.gather( + *(port_client.create_page(page) for page in defaults.pages), + return_exceptions=True, + ) + + created_pages = [ + result.get("identifier", "") + for result in create_pages_result + if not isinstance(result, BaseException) + ] + + pages_errors = [ + result for result in create_pages_result if isinstance(result, Exception) + ] + + if pages_errors: + for error in pages_errors: + if isinstance(error, httpx.HTTPStatusError): + logger.warning( + f"Failed to create resources: {error.response.text}. Rolling back changes..." + ) + + raise AbortDefaultCreationError( + created_blueprints, pages_errors, created_pages + ) + await port_client.create_integration( integration_config.integration.type, integration_config.event_listener.to_request(), port_app_config=defaults.port_app_config, ) - except httpx.HTTPStatusError as e: + except httpx.HTTPStatusError as err: logger.error( - f"Failed to create resources: {e.response.text}. Rolling back changes..." + f"Failed to create resources: {err.response.text}. Rolling back changes..." ) - raise AbortDefaultCreationError(created_blueprints, [e]) + raise AbortDefaultCreationError(created_blueprints, [err], created_pages) async def _initialize_defaults( @@ -133,6 +189,7 @@ async def _initialize_defaults( return None try: + logger.info("Found default resources, starting creation process") await _create_resources(port_client, defaults, integration_config) except AbortDefaultCreationError as e: logger.warning( @@ -148,6 +205,18 @@ async def _initialize_defaults( for identifier in e.blueprints_to_rollback ) ) + if e.pages_to_rollback: + logger.warning( + f"Failed to create resources. Rolling back pages : {e.pages_to_rollback}" + ) + await asyncio.gather( + *( + port_client.delete_page( + identifier, + ) + for identifier in e.pages_to_rollback + ) + ) raise ExceptionGroup(str(e), e.errors) diff --git a/port_ocean/core/handlers/entities_state_applier/port/applier.py b/port_ocean/core/handlers/entities_state_applier/port/applier.py index d7330286e3..8e27d60e47 100644 --- a/port_ocean/core/handlers/entities_state_applier/port/applier.py +++ b/port_ocean/core/handlers/entities_state_applier/port/applier.py @@ -130,6 +130,7 @@ async def apply_diff( logger.info("Upserting modified entities") await self.upsert(diff.modified, user_agent_type) + logger.info("Deleting diff entities") await self._delete_diff( diff.deleted, diff.created + diff.modified, user_agent_type ) @@ -149,6 +150,7 @@ async def delete_diff( ) await self._validate_entity_diff(diff) + logger.info("Deleting diff entities") await self._delete_diff( diff.deleted, diff.created + diff.modified, user_agent_type ) diff --git a/port_ocean/core/integrations/mixins/sync_raw.py b/port_ocean/core/integrations/mixins/sync_raw.py index 398cca9900..61c7c28b2d 100644 --- a/port_ocean/core/integrations/mixins/sync_raw.py +++ b/port_ocean/core/integrations/mixins/sync_raw.py @@ -352,6 +352,7 @@ async def sync_raw_all( except asyncio.CancelledError as e: logger.warning("Resync aborted successfully") else: + logger.info("Starting resync diff calculation") flat_created_entities, errors = zip_and_sum(creation_results) or [ [], [], @@ -368,6 +369,9 @@ async def sync_raw_all( logger.error(message, exc_info=error_group) else: + logger.info( + f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}" + ) await self.entities_state_applier.delete_diff( {"before": entities_at_port, "after": flat_created_entities}, user_agent_type, diff --git a/port_ocean/exceptions/port_defaults.py b/port_ocean/exceptions/port_defaults.py index 4ab535df43..177e79d11b 100644 --- a/port_ocean/exceptions/port_defaults.py +++ b/port_ocean/exceptions/port_defaults.py @@ -2,8 +2,14 @@ class AbortDefaultCreationError(BaseOceanException): - def __init__(self, blueprints_to_rollback: list[str], errors: list[Exception]): + def __init__( + self, + blueprints_to_rollback: list[str], + errors: list[Exception], + pages_to_rollback: list[str] | None = None, + ): self.blueprints_to_rollback = blueprints_to_rollback + self.pages_to_rollback = pages_to_rollback self.errors = errors super().__init__("Aborting defaults creation") diff --git a/pyproject.toml b/pyproject.toml index 84f5ee1f52..6529aacff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.4.12" +version = "0.4.13" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"