From 80d7584c069c56a342a770c1b4752bac9b7a4f2b Mon Sep 17 00:00:00 2001 From: maha-hajja Date: Thu, 7 Mar 2024 22:10:38 -0800 Subject: [PATCH 1/8] update pipeline config files section --- .../specifications.mdx | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index d16fd2c0..580b7615 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -43,20 +43,21 @@ pipelines: # A list of pipeline configurations name: my-file-destination settings: path: ./file2.txt - processors: # A list of processor configurations, processors are attached to the connector - - id: proc1 # Processor ID [required] - type: js # Processor type [required] - workers: 2 # Number of parallel workers - settings: # A map of configuration keys and values for the processor (specific to the chosen type) + processors: # A list of processor configurations, processors are attached to the connector + - id: proc1 # Processor ID [required] + plugin: custom.javascript # Processor plugin name [required] + workers: 2 # Number of parallel workers + settings: # A map of configuration keys and values for the processor (specific to the chosen type) script: > function process(record) { return record; // pass through } - processors: # A list of processor configurations, processors are attached to the pipeline - - id: proc2 # Processor ID [required] - type: hoistfieldpayload # Processor type [required] - settings: # A map of configuration keys and values for the processor (specific to the chosen type) - field: ${ENV_VAR} # You can use environment variables by wrapping them in a dollar sign and curly braces ${} + processors: # A list of processor configurations, processors are attached to the pipeline + - id: proc2 # Processor ID [required] + plugin: field.set # Processor type [required] + settings: # A map of configuration keys and values for the processor (specific to the chosen type) + field: .Payload.After.key # You can use environment variables by wrapping them in a dollar sign and curly braces ${} + value: ${ENV_VAR} # You can use environment variables by wrapping them in a dollar sign and curly braces ${} dead-letter-queue: # Dead-letter queue (DLQ) configuration plugin: "builtin:file" # DLQ Connector plugin settings: # A map of configuration keys and values for the plugin (specific to the chosen plugin) @@ -267,7 +268,7 @@ beginning. - **Default**: None - **Description**: This node contains the configuration of a single processor. -### id +### plugin - **Data Type**: String - **Required**: Yes @@ -276,14 +277,15 @@ beginning. identify the processor inside the pipeline. If multiple processors inside the pipeline use the same ID, provisioning will fail. -### type +### plugin - **Data Type**: String - **Required**: Yes - **Default**: None -- **Description**: Defines the processor type (e.g. `js`). Check out the - [processors documentation](/docs/processors/getting-started) to find out which - processors are supported. +- **Description**: Defines the processor's plugin name (e.g. `field.set`). Check out the + [processors documentation](/docs/processors/builtin) to find the list of builtin processors + we provide. Also, check [Building processors](/docs/processors/building-processors) + for how to build and use your own WASM processor. ### settings @@ -291,7 +293,7 @@ beginning. - **Required**: Yes - **Default**: None - **Description**: A map of configuration keys and values for the processor. The - values in this map depend on the chosen processor [`type`](#type-1). + values in this map depend on the chosen processor [`plugin`](#plugin-1). ### workers From 13fe83a7d2cb19d65fa0f112f382d961f9f44f8a Mon Sep 17 00:00:00 2001 From: maha-hajja Date: Thu, 7 Mar 2024 22:18:38 -0800 Subject: [PATCH 2/8] update type to plugin --- docs/pipeline-configuration-files/specifications.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index 580b7615..6daa116a 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -47,7 +47,7 @@ pipelines: # A list of pipeline configurations - id: proc1 # Processor ID [required] plugin: custom.javascript # Processor plugin name [required] workers: 2 # Number of parallel workers - settings: # A map of configuration keys and values for the processor (specific to the chosen type) + settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) script: > function process(record) { return record; // pass through @@ -55,8 +55,8 @@ pipelines: # A list of pipeline configurations processors: # A list of processor configurations, processors are attached to the pipeline - id: proc2 # Processor ID [required] plugin: field.set # Processor type [required] - settings: # A map of configuration keys and values for the processor (specific to the chosen type) - field: .Payload.After.key # You can use environment variables by wrapping them in a dollar sign and curly braces ${} + settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) + field: .Payload.After.key value: ${ENV_VAR} # You can use environment variables by wrapping them in a dollar sign and curly braces ${} dead-letter-queue: # Dead-letter queue (DLQ) configuration plugin: "builtin:file" # DLQ Connector plugin From e26e7fa067bbe0bddd24a131d5a06efd02b9e796 Mon Sep 17 00:00:00 2001 From: maha-hajja Date: Mon, 11 Mar 2024 11:32:33 -0700 Subject: [PATCH 3/8] add condition --- docs/pipeline-configuration-files/specifications.mdx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index 6daa116a..f7db90d5 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -46,6 +46,7 @@ pipelines: # A list of pipeline configurations processors: # A list of processor configurations, processors are attached to the connector - id: proc1 # Processor ID [required] plugin: custom.javascript # Processor plugin name [required] + condition: "{{ eq .Metadata.foo "bar" }}" # Condition (Go template expression) that dictates if the record will be passed to the processor or not. workers: 2 # Number of parallel workers settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) script: > @@ -268,7 +269,7 @@ beginning. - **Default**: None - **Description**: This node contains the configuration of a single processor. -### plugin +### id - **Data Type**: String - **Required**: Yes @@ -287,6 +288,15 @@ beginning. we provide. Also, check [Building processors](/docs/processors/building-processors) for how to build and use your own WASM processor. +### condition + +- **Data Type**: String +- **Required**: No +- **Default**: `True` (all records will passed to the processor) +- **Description**: a Go template expression that will be used as a condition to pass the record to the processor or not. +the Go template expression will be evaluated, and the evaluated value should either be `True` or `False`, `True` means +the record will be passed to the processor, `False` means it will be skipped. + ### settings - **Data Type**: Mapping node From 0fa9bca37ffd01e0254894088562865814009988 Mon Sep 17 00:00:00 2001 From: maha-hajja Date: Mon, 11 Mar 2024 11:34:19 -0700 Subject: [PATCH 4/8] delete referencing the "building processor" doc --- docs/pipeline-configuration-files/specifications.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index f7db90d5..bf6c8caa 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -285,8 +285,7 @@ beginning. - **Default**: None - **Description**: Defines the processor's plugin name (e.g. `field.set`). Check out the [processors documentation](/docs/processors/builtin) to find the list of builtin processors - we provide. Also, check [Building processors](/docs/processors/building-processors) - for how to build and use your own WASM processor. + we provide. ### condition From 4e1b22ec34e00759ca74429bba40e0904e9eac6b Mon Sep 17 00:00:00 2001 From: Maha Hajja <82542081+maha-hajja@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:34:50 -0700 Subject: [PATCH 5/8] Update docs/pipeline-configuration-files/specifications.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lovro Mažgon --- docs/pipeline-configuration-files/specifications.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index bf6c8caa..8c14a90b 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -292,9 +292,9 @@ beginning. - **Data Type**: String - **Required**: No - **Default**: `True` (all records will passed to the processor) -- **Description**: a Go template expression that will be used as a condition to pass the record to the processor or not. -the Go template expression will be evaluated, and the evaluated value should either be `True` or `False`, `True` means -the record will be passed to the processor, `False` means it will be skipped. +- **Description**: A [Go template expression](https://pkg.go.dev/text/template) that will be used as a condition to pass the record to the processor or not. +The Go template expression will be evaluated using each record and needs to produce `true` or `false`. If it produces `true` +the record will be passed to the processor, `false` means it will continue down the pipeline without getting processed. ### settings From 961bc16d51bef2bfc1207becf14c78527e458556 Mon Sep 17 00:00:00 2001 From: Maha Hajja <82542081+maha-hajja@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:34:59 -0700 Subject: [PATCH 6/8] Update docs/pipeline-configuration-files/specifications.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lovro Mažgon --- docs/pipeline-configuration-files/specifications.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index 8c14a90b..d595192b 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -291,7 +291,7 @@ beginning. - **Data Type**: String - **Required**: No -- **Default**: `True` (all records will passed to the processor) +- **Default**: `true` (all records will be passed to the processor) - **Description**: A [Go template expression](https://pkg.go.dev/text/template) that will be used as a condition to pass the record to the processor or not. The Go template expression will be evaluated using each record and needs to produce `true` or `false`. If it produces `true` the record will be passed to the processor, `false` means it will continue down the pipeline without getting processed. From 376643a7a8dcec43b5cea8d186a8afa3be6aafc6 Mon Sep 17 00:00:00 2001 From: Maha Hajja <82542081+maha-hajja@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:35:16 -0700 Subject: [PATCH 7/8] Update docs/pipeline-configuration-files/specifications.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lovro Mažgon --- docs/pipeline-configuration-files/specifications.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index d595192b..55d5d172 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -302,7 +302,7 @@ the record will be passed to the processor, `false` means it will continue down - **Required**: Yes - **Default**: None - **Description**: A map of configuration keys and values for the processor. The - values in this map depend on the chosen processor [`plugin`](#plugin-1). + values in this map depend on the chosen processor [`plugin`](#plugin-2). ### workers From f53b1f9046bc6bf691ac025124c4eaa236659ae6 Mon Sep 17 00:00:00 2001 From: Maha Hajja <82542081+maha-hajja@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:04:33 -0700 Subject: [PATCH 8/8] Update docs/pipeline-configuration-files/specifications.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lovro Mažgon --- docs/pipeline-configuration-files/specifications.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index 55d5d172..bb7e7e86 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -46,7 +46,7 @@ pipelines: # A list of pipeline configurations processors: # A list of processor configurations, processors are attached to the connector - id: proc1 # Processor ID [required] plugin: custom.javascript # Processor plugin name [required] - condition: "{{ eq .Metadata.foo "bar" }}" # Condition (Go template expression) that dictates if the record will be passed to the processor or not. + condition: '{{ eq .Metadata.foo "bar" }}' # Condition (Go template expression) that dictates if the record will be passed to the processor or not. workers: 2 # Number of parallel workers settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) script: >