From 8efa10261ce577bac36e6f10e91d43e59ccf059c Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 15 Nov 2023 19:41:56 +0200 Subject: [PATCH 1/6] fixed default mapping and added Control the payload of your self-service experiences README.md --- README.md | 341 ++++++++++++++++++++++++++++ app/control_the_payload_config.json | 4 +- 2 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..14af1d0 --- /dev/null +++ b/README.md @@ -0,0 +1,341 @@ +# Port Agent + +## [Documentation](https://docs.getport.io/create-self-service-experiences/setup-backend/port-execution-agent/) + +## Control the payload of your self-service experiences + +Some of the 3rd party application that you may want to integrate with may not accept the raw payload incoming from port +self-service actions. The port agent allows you to control the payload that is sent to the 3rd party application. + +By mounting a control the payload mapping file to the port agent, you can control the request that is sent to your 3rd +party application. + +### Control the payload mapping + +The payload mapping file is a json file that contains the mapping of the payload that is sent to the port agent to the +payload that is sent to the 3rd party application. + +The payload mapping file is mounted to the port agent as a volume. The path to the payload mapping file is set in the +`CONTROL_THE_PAYLOAD_CONFIG_PATH` environment variable. By default, the port agent will look for the payload mapping +file +at `~/control_the_payload_config.json`. + +The payload mapping file is a json file that contains a list of mappings. Each mapping contains the request fields that +will be overridden and sent to the 3rd party application. + +Each of the mapping fields can be constructed by JQ expressions. The JQ expression will be evaluated against the +original payload that is sent to the port agent from Port and the result will be sent to the 3rd party application. + +The mapping file schema is as follows: + +``` +[ # Can have multiple mappings. Will use the first one it will find with enabled = True (Allows you to apply mapping over multiple actions at once) + { + "enabled": bool || JQ, + "url": JQ, # Optional. default is the incoming url from port + "method": JQ, # Optional. default is POST. Should reutnr one of the following string values POST / PUT / DELETE / GET + "headers": dict[str, JQ], # Optional. default is {} + "body": ".body", # Optional. default is the whole payload incoming from Port. + "query": dict[str, JQ] # Optional. default is {} + } +] +``` + +**The body can be partially constructed by json as follows:** + +```json +{ + "body": { + "key": 2, + "key2": { + "key3": ".im.a.jq.expression", + "key4": "\"im a string\"" + } + } +} +``` + +### The incoming message to base your mapping on + +
+Example for incoming event + +```json +{ + "action": "action_identifier", + "resourceType": "run", + "status": "TRIGGERED", + "trigger": { + "by": { + "orgId": "org_XXX", + "userId": "auth0|XXXXX", + "user": { + "email": "executor@mail.com", + "firstName": "user", + "lastName": "userLastName", + "phoneNumber": "0909090909090909", + "picture": "https://s.gravatar.com/avatar/dd1cf547c8b950ce6966c050234ac997?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fga.png", + "providers": [ + "port" + ], + "status": "ACTIVE", + "id": "auth0|XXXXX", + "createdAt": "2022-12-08T16:34:20.735Z", + "updatedAt": "2023-11-13T15:11:38.243Z" + } + }, + "origin": "UI", + "at": "2023-11-13T15:20:16.641Z" + }, + "context": { + "entity": "e_iQfaF14FJln6GVBn", + "blueprint": "kubecostCloudAllocation", + "runId": "r_HardNzG6kzc9vWOQ" + }, + "payload": { + "entity": { + "identifier": "e_iQfaF14FJln6GVBn", + "title": "myEntity", + "icon": "Port", + "blueprint": "myBlueprint", + "team": [], + "properties": { + }, + "relations": {}, + "createdAt": "2023-11-13T15:24:46.880Z", + "createdBy": "auth0|XXXXX", + "updatedAt": "2023-11-13T15:24:46.880Z", + "updatedBy": "auth0|XXXXX" + }, + "action": { + "invocationMethod": { + "type": "WEBHOOK", + "agent": true, + "synchronized": false, + "method": "POST", + "url": "https://myGitlabHost.com" + }, + "trigger": "DAY-2" + }, + "properties": { + }, + "censoredProperties": [] + } +} +``` + +
+ +### Examples + +#### Terraform Cloud + +Create the following blueprint, action and mapping to trigger a Terraform Cloud run. + +
+Blueprint + +```json +{ + "identifier": "terraform_cloud_workspace", + "title": "Terraform Cloud Workspace", + "icon": "Terraform", + "schema": { + "properties": { + "workspace_id": { + "title": "Workspace Id", + "type": "string" + } + }, + "required": [ + "workspace_id" + ] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": {} +} +``` +
+ +
+Action + +```json +[ + { + "identifier": "trigger_tf_run", + "title": "Trigger TF Cloud run", + "icon": "Terraform", + "userInputs": { + "properties": {}, + "required": [], + "order": [] + }, + "invocationMethod": { + "type": "WEBHOOK", + "agent": true, + "synchronized": false, + "method": "POST", + "url": "https://app.terraform.io/api/v2/runs/" + }, + "trigger": "DAY-2", + "requiredApproval": false + } +] +``` +
+ +
+Mapping + +```json +{ + "enabled": ".action == \"trigger_tf_run\"", + "headers": { + "Authorization": "\"Bearer \" + env.TF_TOKEN", + "Content-Type": "\"application/vnd.api+json\"" + }, + "body": { + "data": { + "attributes": { + "is-destroy": false, + "message": "\"Triggered via Port\"", + "variables": ".payload.properties | to_entries | map({key: .key, value: .value})" + }, + "type": "\"runs\"", + "relationships": { + "workspace": { + "data": { + "type": "\"workspaces\"", + "id": ".payload.entity.properties.workspaceId" + } + } + } + } + } + } +``` +
+ +**Port agent installation**: + +```sh +helm repo add port-labs https://port-labs.github.io/helm-charts + +helm repo update + +helm install my-port-agent port-labs/port-agent \ + --create-namespace --namespace port-agent \ + --set env.normal.PORT_ORG_ID=YOUR_ORG_ID \ + --set env.normal.KAFKA_CONSUMER_GROUP_ID=YOUR_KAFKA_CONSUMER_GROUP \ + --set env.secret.KAFKA_CONSUMER_USERNAME=YOUR_KAFKA_USERNAME \ + --set env.secret.KAFKA_CONSUMER_PASSWORD=YOUR_KAFKA_PASSWORD + --set env.normal.KAFKA_CONSUMER_BROKERS=PORT_KAFKA_BROKERS \ + --set env.normal.STREAMER_NAME=KAFKA \ + --set env.normal.KAFKA_CONSUMER_AUTHENTICATION_MECHANISM=SCRAM-SHA-512 \ + --set env.normal.KAFKA_CONSUMER_AUTO_OFFSET_RESET=earliest \ + --set env.normal.KAFKA_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ + --set env.secret.TF_TOKEN=YOU_TERRAFORM_CLOUD_TOKEN \ + --set controlThePayloadConfigFile=invocations.json +``` + + + +#### CircleCI + +Create the following blueprint, action and mapping to trigger a CircleCI pipeline. + +
+Blueprint + +```json +{ + "identifier": "circle_ci_project", + "title": "CircleCI Project", + "icon": "CircleCI", + "schema": { + "properties": { + "project_slug": { + "title": "Slug", + "type": "string" + } + }, + "required": [ + "project_slug" + ] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "relations": {} +} +``` +
+ +
+Action + +```json +[ + { + "identifier": "trigger_circle_ci_pipeline", + "title": "Trigger CircleCI pipeline", + "icon": "CircleCI", + "userInputs": { + "properties": {}, + "required": [], + "order": [] + }, + "invocationMethod": { + "type": "WEBHOOK", + "agent": true, + "synchronized": false, + "method": "POST", + "url": "https://circleci.com" + }, + "trigger": "DAY-2", + "requiredApproval": false + } +] +``` +
+ +
+Mapping + +```json +{ + "enabled": ".action == \"trigger_circle_ci_pipeline\"", + "url": "(env.CIRCLE_CI_URL // \"https://circleci.com\") as $baseUrl | .payload.entity.properties.project_slug | @uri as $path | $baseUrl + \"/api/v2/project/\" + $path + \"/pipeline\"", + "headers": { + "Circle-Token": "env.CIRCLE_CI_TOKEN" + }, + "body": { + "branch": ".payload.properties.branch // \"main\"", + "parameters": ".payload.action.invocationMethod as $invocationMethod | .payload.properties | to_entries | map({(.key): (.value | tostring)}) | add | if $invocationMethod.omitUserInputs then {} else . end" + } + } +``` +
+ +**Port agent installation**: + +```sh +helm repo add port-labs https://port-labs.github.io/helm-charts + +helm repo update + +helm install my-port-agent port-labs/port-agent \ + --create-namespace --namespace port-agent \ + --set env.normal.PORT_ORG_ID=YOUR_ORG_ID \ + --set env.normal.KAFKA_CONSUMER_GROUP_ID=YOUR_KAFKA_CONSUMER_GROUP \ + --set env.secret.KAFKA_CONSUMER_USERNAME=YOUR_KAFKA_USERNAME \ + --set env.secret.KAFKA_CONSUMER_PASSWORD=YOUR_KAFKA_PASSWORD + --set env.normal.KAFKA_CONSUMER_BROKERS=PORT_KAFKA_BROKERS \ + --set env.normal.STREAMER_NAME=KAFKA \ + --set env.normal.KAFKA_CONSUMER_AUTHENTICATION_MECHANISM=SCRAM-SHA-512 \ + --set env.normal.KAFKA_CONSUMER_AUTO_OFFSET_RESET=earliest \ + --set env.normal.KAFKA_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ + --set env.secret.CIRCLE_CI_TOKEN=YOUR_CIRCLE_CI_PERSONAL_TOKEN \ + --set controlThePayloadConfigFile=invocations.json +``` \ No newline at end of file diff --git a/app/control_the_payload_config.json b/app/control_the_payload_config.json index 9d2a940..af87190 100644 --- a/app/control_the_payload_config.json +++ b/app/control_the_payload_config.json @@ -1,10 +1,10 @@ [ { "enabled": ".payload.action.invocationMethod.type == \"GITLAB\"", - "url": "($ENV.GITLAB_URL // \"https://gitlab.com\") as $baseUrl | (.payload.action.invocationMethod.groupName + \"/\" +.payload.action.invocationMethod.projectName) | @uri as $path | $baseUrl + \"/api/v4/projects/\" + $path + \"/trigger/pipeline\"", + "url": "(env.GITLAB_URL // \"https://gitlab.com/\") as $baseUrl | (.payload.action.invocationMethod.groupName + \"/\" +.payload.action.invocationMethod.projectName) | @uri as $path | $baseUrl + \"api/v4/projects/\" + $path + \"/trigger/pipeline\"", "body": { "ref": ".payload.properties.ref // .payload.action.invocationMethod.defaultRef // \"main\"", - "token": ".payload.action.invocationMethod.groupName as $gitlab_group | .payload.action.invocationMethod.projectName as $gitlab_project | $ENV[($gitlab_group + \"_\" + ($gitlab_project | gsub(\"/\"; \"_\")))]", + "token": ".payload.action.invocationMethod.groupName as $gitlab_group | .payload.action.invocationMethod.projectName as $gitlab_project | env[($gitlab_group | gsub(\"/\"; \"_\")) + \"_\" + $gitlab_project]", "variables": ".payload.action.invocationMethod as $invocationMethod | .payload.properties | to_entries | map({(.key): (.value | tostring)}) | add | if $invocationMethod.omitUserInputs then {} else . end", "port_payload": "if .payload.action.invocationMethod.omitPayload then {} else . end" } From 002b847a8e8279e5ff894f5d33c3c86b91d9a5c0 Mon Sep 17 00:00:00 2001 From: yair Date: Wed, 15 Nov 2023 21:13:42 +0200 Subject: [PATCH 2/6] passed the config file with --set-file --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 14af1d0..219d552 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ helm install my-port-agent port-labs/port-agent \ --set env.normal.KAFKA_CONSUMER_AUTO_OFFSET_RESET=earliest \ --set env.normal.KAFKA_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ --set env.secret.TF_TOKEN=YOU_TERRAFORM_CLOUD_TOKEN \ - --set controlThePayloadConfigFile=invocations.json + --set-file controlThePayloadConfig=./invocations.json ``` @@ -337,5 +337,5 @@ helm install my-port-agent port-labs/port-agent \ --set env.normal.KAFKA_CONSUMER_AUTO_OFFSET_RESET=earliest \ --set env.normal.KAFKA_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ --set env.secret.CIRCLE_CI_TOKEN=YOUR_CIRCLE_CI_PERSONAL_TOKEN \ - --set controlThePayloadConfigFile=invocations.json + --set-file controlThePayloadConfig=./invocations.json ``` \ No newline at end of file From 22d15cde6244ab342aaccec058c6d81e5259cea1 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 16 Nov 2023 10:43:14 +0200 Subject: [PATCH 3/6] port-agent control the payload readme updates --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 219d552..08603ab 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,16 @@ ## Control the payload of your self-service experiences -Some of the 3rd party application that you may want to integrate with may not accept the raw payload incoming from port +Some of the 3rd party applications that you may want to integrate with may not accept the raw payload incoming from port self-service actions. The port agent allows you to control the payload that is sent to the 3rd party application. -By mounting a control the payload mapping file to the port agent, you can control the request that is sent to your 3rd -party application. +You can alter the requests sent to your third-party application by mounting the payload mapping config file with the +port-agent container. ### Control the payload mapping -The payload mapping file is a json file that contains the mapping of the payload that is sent to the port agent to the -payload that is sent to the 3rd party application. +The payload mapping file is a JSON file that shows how the information sent to the port agent translated to the +information sent to the third-party application. The payload mapping file is mounted to the port agent as a volume. The path to the payload mapping file is set in the `CONTROL_THE_PAYLOAD_CONFIG_PATH` environment variable. By default, the port agent will look for the payload mapping @@ -304,7 +304,7 @@ Create the following blueprint, action and mapping to trigger a CircleCI pipelin Mapping ```json -{ +{` "enabled": ".action == \"trigger_circle_ci_pipeline\"", "url": "(env.CIRCLE_CI_URL // \"https://circleci.com\") as $baseUrl | .payload.entity.properties.project_slug | @uri as $path | $baseUrl + \"/api/v2/project/\" + $path + \"/pipeline\"", "headers": { From 1063a940b45619f0b9982a088030ca1b49ebdb34 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 16 Nov 2023 11:06:26 +0200 Subject: [PATCH 4/6] port-agent control the payload readme updates --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08603ab..1e2f14d 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,7 @@ Create the following blueprint, action and mapping to trigger a CircleCI pipelin Mapping ```json -{` +{ "enabled": ".action == \"trigger_circle_ci_pipeline\"", "url": "(env.CIRCLE_CI_URL // \"https://circleci.com\") as $baseUrl | .payload.entity.properties.project_slug | @uri as $path | $baseUrl + \"/api/v2/project/\" + $path + \"/pipeline\"", "headers": { From 3a7d025a29654536e39ad87c4b4384dbcb433c53 Mon Sep 17 00:00:00 2001 From: Mor Paz <1912.mor@gmail.com> Date: Thu, 16 Nov 2023 12:40:39 +0200 Subject: [PATCH 5/6] Update README.md --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 1e2f14d..256f39d 100644 --- a/README.md +++ b/README.md @@ -2,38 +2,40 @@ ## [Documentation](https://docs.getport.io/create-self-service-experiences/setup-backend/port-execution-agent/) -## Control the payload of your self-service experiences +## Control the payload of your self-service actions -Some of the 3rd party applications that you may want to integrate with may not accept the raw payload incoming from port -self-service actions. The port agent allows you to control the payload that is sent to the 3rd party application. +Some of the 3rd party applications that you may want to integrate with may not accept the raw payload incoming from Port's +self-service actions. The Port agent allows you to control the payload that is sent to every 3rd party application. -You can alter the requests sent to your third-party application by mounting the payload mapping config file with the -port-agent container. +You can alter the requests sent to your third-party application by providing a payload mapping config file when deploying the +Port-agent container. ### Control the payload mapping -The payload mapping file is a JSON file that shows how the information sent to the port agent translated to the -information sent to the third-party application. +The payload mapping file is a JSON file that specifies how to transform the request sent to the Port agent to the +request that is sent to the third-party application. -The payload mapping file is mounted to the port agent as a volume. The path to the payload mapping file is set in the -`CONTROL_THE_PAYLOAD_CONFIG_PATH` environment variable. By default, the port agent will look for the payload mapping +The payload mapping file is mounted to the Port agent as a volume. The path to the payload mapping file is set in the +`CONTROL_THE_PAYLOAD_CONFIG_PATH` environment variable. By default, the Port agent will look for the payload mapping file at `~/control_the_payload_config.json`. The payload mapping file is a json file that contains a list of mappings. Each mapping contains the request fields that will be overridden and sent to the 3rd party application. +You can see examples showing how to deploy the Port agent with different mapping configurations for various common use cases below. + Each of the mapping fields can be constructed by JQ expressions. The JQ expression will be evaluated against the original payload that is sent to the port agent from Port and the result will be sent to the 3rd party application. -The mapping file schema is as follows: +Here is the mapping file schema: ``` [ # Can have multiple mappings. Will use the first one it will find with enabled = True (Allows you to apply mapping over multiple actions at once) { "enabled": bool || JQ, "url": JQ, # Optional. default is the incoming url from port - "method": JQ, # Optional. default is POST. Should reutnr one of the following string values POST / PUT / DELETE / GET + "method": JQ, # Optional. default is POST. Should return one of the following string values POST / PUT / DELETE / GET "headers": dict[str, JQ], # Optional. default is {} "body": ".body", # Optional. default is the whole payload incoming from Port. "query": dict[str, JQ] # Optional. default is {} @@ -218,7 +220,7 @@ Create the following blueprint, action and mapping to trigger a Terraform Cloud ``` -**Port agent installation**: +**Port agent installation for Terraform cloud example**: ```sh helm repo add port-labs https://port-labs.github.io/helm-charts @@ -230,7 +232,7 @@ helm install my-port-agent port-labs/port-agent \ --set env.normal.PORT_ORG_ID=YOUR_ORG_ID \ --set env.normal.KAFKA_CONSUMER_GROUP_ID=YOUR_KAFKA_CONSUMER_GROUP \ --set env.secret.KAFKA_CONSUMER_USERNAME=YOUR_KAFKA_USERNAME \ - --set env.secret.KAFKA_CONSUMER_PASSWORD=YOUR_KAFKA_PASSWORD + --set env.secret.KAFKA_CONSUMER_PASSWORD=YOUR_KAFKA_PASSWORD \ --set env.normal.KAFKA_CONSUMER_BROKERS=PORT_KAFKA_BROKERS \ --set env.normal.STREAMER_NAME=KAFKA \ --set env.normal.KAFKA_CONSUMER_AUTHENTICATION_MECHANISM=SCRAM-SHA-512 \ @@ -318,7 +320,7 @@ Create the following blueprint, action and mapping to trigger a CircleCI pipelin ``` -**Port agent installation**: +**Port agent installation for CircleCI example**: ```sh helm repo add port-labs https://port-labs.github.io/helm-charts @@ -338,4 +340,4 @@ helm install my-port-agent port-labs/port-agent \ --set env.normal.KAFKA_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ --set env.secret.CIRCLE_CI_TOKEN=YOUR_CIRCLE_CI_PERSONAL_TOKEN \ --set-file controlThePayloadConfig=./invocations.json -``` \ No newline at end of file +``` From a7f687a9f2d6ce9888076ab2d4d883cbec2b28b6 Mon Sep 17 00:00:00 2001 From: yair Date: Thu, 16 Nov 2023 12:59:56 +0200 Subject: [PATCH 6/6] control the payload readme fixes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 256f39d..36c335f 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Create the following blueprint, action and mapping to trigger a Terraform Cloud
-Mapping +Mapping - (Should be saved as `invocations.json`) ```json { @@ -210,7 +210,7 @@ Create the following blueprint, action and mapping to trigger a Terraform Cloud "workspace": { "data": { "type": "\"workspaces\"", - "id": ".payload.entity.properties.workspaceId" + "id": ".payload.entity.properties.workspace_id" } } } @@ -303,7 +303,7 @@ Create the following blueprint, action and mapping to trigger a CircleCI pipelin
-Mapping +Mapping - (Should be saved as `invocations.json`) ```json {