From 678b96060101b0d21c00c133f1dd7183fe97cc49 Mon Sep 17 00:00:00 2001 From: Maha Hajja <82542081+maha-hajja@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:18:33 -0700 Subject: [PATCH] update pipeline config files section (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update pipeline config files section * update type to plugin * add condition * delete referencing the "building processor" doc * Update docs/pipeline-configuration-files/specifications.mdx Co-authored-by: Lovro Mažgon * Update docs/pipeline-configuration-files/specifications.mdx Co-authored-by: Lovro Mažgon * Update docs/pipeline-configuration-files/specifications.mdx Co-authored-by: Lovro Mažgon * Update docs/pipeline-configuration-files/specifications.mdx Co-authored-by: Lovro Mažgon --------- Co-authored-by: Lovro Mažgon --- .../specifications.mdx | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/pipeline-configuration-files/specifications.mdx b/docs/pipeline-configuration-files/specifications.mdx index d16fd2c0..bb7e7e86 100644 --- a/docs/pipeline-configuration-files/specifications.mdx +++ b/docs/pipeline-configuration-files/specifications.mdx @@ -43,20 +43,22 @@ pipelines: # A list of pipeline configurations name: my-file-destination settings: path: ./file2.txt - processors: # A list of processor configurations, processors are attached to the connector - - id: proc1 # Processor ID [required] - type: js # Processor type [required] - workers: 2 # Number of parallel workers - settings: # A map of configuration keys and values for the processor (specific to the chosen type) + processors: # A list of processor configurations, processors are attached to the connector + - id: proc1 # Processor ID [required] + plugin: custom.javascript # Processor plugin name [required] + condition: '{{ eq .Metadata.foo "bar" }}' # Condition (Go template expression) that dictates if the record will be passed to the processor or not. + workers: 2 # Number of parallel workers + settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) script: > function process(record) { return record; // pass through } - processors: # A list of processor configurations, processors are attached to the pipeline - - id: proc2 # Processor ID [required] - type: hoistfieldpayload # Processor type [required] - settings: # A map of configuration keys and values for the processor (specific to the chosen type) - field: ${ENV_VAR} # You can use environment variables by wrapping them in a dollar sign and curly braces ${} + processors: # A list of processor configurations, processors are attached to the pipeline + - id: proc2 # Processor ID [required] + plugin: field.set # Processor type [required] + settings: # A map of configuration keys and values for the processor (specific to the chosen processor plugin) + field: .Payload.After.key + value: ${ENV_VAR} # You can use environment variables by wrapping them in a dollar sign and curly braces ${} dead-letter-queue: # Dead-letter queue (DLQ) configuration plugin: "builtin:file" # DLQ Connector plugin settings: # A map of configuration keys and values for the plugin (specific to the chosen plugin) @@ -276,14 +278,23 @@ beginning. identify the processor inside the pipeline. If multiple processors inside the pipeline use the same ID, provisioning will fail. -### type +### plugin - **Data Type**: String - **Required**: Yes - **Default**: None -- **Description**: Defines the processor type (e.g. `js`). Check out the - [processors documentation](/docs/processors/getting-started) to find out which - processors are supported. +- **Description**: Defines the processor's plugin name (e.g. `field.set`). Check out the + [processors documentation](/docs/processors/builtin) to find the list of builtin processors + we provide. + +### condition + +- **Data Type**: String +- **Required**: No +- **Default**: `true` (all records will be passed to the processor) +- **Description**: A [Go template expression](https://pkg.go.dev/text/template) that will be used as a condition to pass the record to the processor or not. +The Go template expression will be evaluated using each record and needs to produce `true` or `false`. If it produces `true` +the record will be passed to the processor, `false` means it will continue down the pipeline without getting processed. ### settings @@ -291,7 +302,7 @@ beginning. - **Required**: Yes - **Default**: None - **Description**: A map of configuration keys and values for the processor. The - values in this map depend on the chosen processor [`type`](#type-1). + values in this map depend on the chosen processor [`plugin`](#plugin-2). ### workers