From 1547498580fba924d29f61e7bb447ee224ad4ad0 Mon Sep 17 00:00:00 2001 From: omby8888 <160610297+omby8888@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:22:00 +0200 Subject: [PATCH] [Framework] Added support for items to parse in mapping (#429) # Description What - A valid mapping can be looked as follows: ```yaml - kind: issue selector: query: .item.name != 'test-item' and .issueType == 'Bug' port: itemsToParse: .fields.components entity: mappings: identifier: .item.name title: .item.name blueprint: '"testBlueprint"' properties: {} relations: issue: .key ``` issue's json from api: ```json { "url": "https://example.com/issue/1", "status": "Open", "issueType": "Bug", "components": [{"name": "Authentication"}, {"name": "Frontend"}], "assignee": "user1", "reporter": "user2", "creator": "user3", "priority": "High", "created": "2024-03-18T10:00:00Z", "updated": "2024-03-18T12:30:00Z", "key": "ISSUE-1", } ``` it will map the component attribute array of each issue got from the api to "testBlueprint" entity, with identifier and parser equals to the component's name Why - To let integrations map internal array to several entities ## Type of change Please leave one option from the following and delete the rest: - [ ] Bug fix (non-breaking change which fixes an issue) - [X] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation) ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --- CHANGELOG.md | 8 +++ .../framework/features/resource-mapping.md | 70 +++++++++++++++---- .../entity_processor/jq_entity_processor.py | 68 +++++++++++++++--- .../core/handlers/port_app_config/models.py | 1 + pyproject.toml | 2 +- 5 files changed, 126 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e449f3d1..2350fa7d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.5.7 (2024-03-19) + + +### Features + +- Added the ability to map entities from raw array attributes by introducing `itemsToParse` key in the mapping configuration + + ## 0.5.6 (2024-03-17) diff --git a/docs/framework-guides/docs/framework/features/resource-mapping.md b/docs/framework-guides/docs/framework/features/resource-mapping.md index da0de706e0..584d41fdf6 100644 --- a/docs/framework-guides/docs/framework/features/resource-mapping.md +++ b/docs/framework-guides/docs/framework/features/resource-mapping.md @@ -121,8 +121,6 @@ how you specify which entities and which properties you want to fill with data f query: .name | startswith("service") ``` - - etc. - - The `port`, `entity` and the `mappings` keys open the section used to map the 3rd-party application object fields to Port entities. To create multiple mappings of the same kind, you can add another item to the `resources` array; @@ -154,7 +152,52 @@ how you specify which entities and which properties you want to fill with data f Pay attention to the value of the `blueprint` key, if you want to use a hardcoded string, you need to encapsulate it in 2 sets of quotes, for example use a pair of single-quotes (`'`) and then another pair of double-quotes (`"`) ::: +- The `itemsToParse` key makes it possible to create multiple entities from a single array attribute of a 3rd-party application object. +In order to reference an array item attribute, use the `.item` JQ expression prefix. +Here is an example mapping configuration that uses the `itemsToParse` syntax with an `issue` kind provided an Ocean Jira integration: +```yaml + - kind: issue + selector: + query: .item.name != 'test-item' and .issueType == 'Bug' + port: + itemsToParse: .fields.comments + entity: + mappings: + identifier: .item.id + blueprint: '"comment"' + properties: + text: .item.text + relations: + issue: .key +``` +Here is a sample JSON object (3rd-party response) that the mapping will be used for: + +```json +{ + "url": "https://example.com/issue/1", + "status": "Open", + "issueType": "Bug", + "comments": [ + { + "id": "123", + "text": "This issue is not reproducing" + }, + { + "id": "456", + "text": "Great issue!" + } + ], + "assignee": "user1", + "reporter": "user2", + "creator": "user3", + "priority": "High", + "created": "2024-03-18T10:00:00Z", + "updated": "2024-03-18T12:30:00Z", + "key": "ISSUE-1" +} +``` +The result of the mapping will be multiple `comment` entities, based on the items from the `comments` array in the JSON. #### Advanced Fields The Ocean framework supports additional flags to provide additional configuration, making it easier to configure its @@ -204,18 +247,19 @@ automatically in cases where the target related entity does not exist in the sof The following table specifies all of the fields that can be specified in the resource mapping configuration: -| Field | Type | Default | Description | -| --------------------------------------- | ----- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `deleteDependentEntities` | bool | `false` | Delete dependent entities when the parent entity is deleted. | -| `createMissingRelatedEntities` | bool | `false` | Create missing related entities when the child entity is created. | -| `resources` | array | `[]` | A list of resources to map. | +| Field | Type | Default | Description | +|-----------------------------------------| ----- | ------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `deleteDependentEntities` | bool | `false` | Delete dependent entities when the parent entity is deleted. | +| `createMissingRelatedEntities` | bool | `false` | Create missing related entities when the child entity is created. | +| `resources` | array | `[]` | A list of resources to map. | | `resources.[].kind`\* | str | | The kind name of the resource. (Should match one of the available kinds in the [integration specification](../../develop-an-integration/integration-spec-and-default-resources.md#features---integration-feature-specification)) | -| `resources.[].selector.query`\* | str | | A JQ expression that will be used to filter the raw data from the 3rd-party application. | -| `resources.[].port.entity.identifier`\* | str | | A JQ expression that will be used to extract the entity identifier. | -| `resources.[].port.entity.blueprint`\* | str | | A JQ expression that will be used to extract the entity blueprint. | -| `resources.[].port.entity.title` | str | | A JQ expression that will be used to extract the entity title. | -| `resources.[].port.entity.properties` | dict | `{}` | An object of property identifier to JQ expressions that will be used to extract the entity properties. | -| `resources.[].port.entity.relations` | dict | `{}` | An object of relation identifier to JQ expressions that will be used to extract the entity properties. | +| `resources.[].selector.query`\* | str | | A JQ expression that will be used to filter the raw data from the 3rd-party application. | +| `resources.[].port.itemsToParse` | str | | A JQ expression that will be used to apply the mapping on the items of an array and generate multiple entities from the array items. | +| `resources.[].port.entity.identifier`\* | str | | A JQ expression that will be used to extract the entity identifier. | +| `resources.[].port.entity.blueprint`\* | str | | A JQ expression that will be used to extract the entity blueprint. | +| `resources.[].port.entity.title` | str | | A JQ expression that will be used to extract the entity title. | +| `resources.[].port.entity.properties` | dict | `{}` | An object of property identifier to JQ expressions that will be used to extract the entity properties. | +| `resources.[].port.entity.relations` | dict | `{}` | An object of relation identifier to JQ expressions that will be used to extract the entity properties. | ## Specify custom resource mapping fields diff --git a/port_ocean/core/handlers/entity_processor/jq_entity_processor.py b/port_ocean/core/handlers/entity_processor/jq_entity_processor.py index aab03f9301..fadcfafa9f 100644 --- a/port_ocean/core/handlers/entity_processor/jq_entity_processor.py +++ b/port_ocean/core/handlers/entity_processor/jq_entity_processor.py @@ -2,6 +2,7 @@ import functools from functools import lru_cache from typing import Any +from loguru import logger import pyjq as jq # type: ignore @@ -67,25 +68,74 @@ async def _search_as_object( return result + async def _get_entity_if_passed_selector( + self, + data: dict[str, Any], + raw_entity_mappings: dict[str, Any], + selector_query: str, + ) -> dict[str, Any]: + should_run = await self._search_as_bool(data, selector_query) + if should_run: + return await self._search_as_object(data, raw_entity_mappings) + return {} + + async def _calculate_entity( + self, + data: dict[str, Any], + raw_entity_mappings: dict[str, Any], + items_to_parse: str | None, + selector_query: str, + ) -> list[dict[str, Any]]: + if items_to_parse: + items = await self._search(data, items_to_parse) + if isinstance(items, list): + return await asyncio.gather( + *[ + self._get_entity_if_passed_selector( + {"item": item, **data}, + raw_entity_mappings, + selector_query, + ) + for item in items + ] + ) + logger.warning( + f"Failed to parse items for JQ expression {items_to_parse}, Expected list but got {type(items)}." + f" Skipping..." + ) + else: + return [ + await self._get_entity_if_passed_selector( + data, raw_entity_mappings, selector_query + ) + ] + return [{}] + async def _calculate_entities( self, mapping: ResourceConfig, raw_data: list[dict[str, Any]] ) -> list[Entity]: - async def calculate_raw(data: dict[str, Any]) -> dict[str, Any]: - should_run = await self._search_as_bool(data, mapping.selector.query) - if should_run and mapping.port.entity: - return await self._search_as_object( - data, mapping.port.entity.mappings.dict(exclude_unset=True) + raw_entity_mappings: dict[str, Any] = mapping.port.entity.mappings.dict( + exclude_unset=True + ) + entities_tasks = [ + asyncio.create_task( + self._calculate_entity( + data, + raw_entity_mappings, + mapping.port.items_to_parse, + mapping.selector.query, ) - return {} - - entities_tasks = [asyncio.create_task(calculate_raw(data)) for data in raw_data] + ) + for data in raw_data + ] entities = await asyncio.gather(*entities_tasks) return [ Entity.parse_obj(entity_data) + for flatten in entities for entity_data in filter( lambda entity: entity.get("identifier") and entity.get("blueprint"), - entities, + flatten, ) ] diff --git a/port_ocean/core/handlers/port_app_config/models.py b/port_ocean/core/handlers/port_app_config/models.py index 725f7a1c1c..a608d293a9 100644 --- a/port_ocean/core/handlers/port_app_config/models.py +++ b/port_ocean/core/handlers/port_app_config/models.py @@ -19,6 +19,7 @@ class MappingsConfig(BaseModel): mappings: EntityMapping entity: MappingsConfig + items_to_parse: str | None = Field(alias="itemsToParse") class Selector(BaseModel): diff --git a/pyproject.toml b/pyproject.toml index 25188bd617..681190429d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.5.6" +version = "0.5.7" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"