[pfsense] Generate processor tags and normalize error handler (#15566)

- Generate tags for processors missing tags
- Normalize the pipeline error handler

Author: taylor-swanson

packages/pfsense/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.2"
+  changes:
+    - description: Generate processor tags and normalize error handler.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15566
 - version: "1.23.1"
   changes:
     - description: Changed owners.

packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -2,27 +2,34 @@
 description: Pipeline for PFsense
 processors:
   - set:
+      tag: set_ecs_version_f5923549
       field: ecs.version
       value: '8.17.0'
   - set:
+      tag: set_observer_vendor_ad9d35cc
       field: observer.vendor
       value: netgate
   - set:
+      tag: set_observer_type_5dddf3ba
       field: observer.type
       value: firewall
   - rename:
+      tag: rename_message_to_event_original_56a77271
       field: message
       target_field: event.original
       ignore_missing: true
       if: ctx.event?.original == null
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - set:
+      tag: set_event_timezone_4ca44cac
       field: event.timezone
       value: "{{{_tmp.tz_offset}}}"
       if: ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'
   - grok:
+      tag: grok_event_original_27d9c8c7
       description: Parse syslog header
       field: event.original
       patterns:
@@ -41,12 +48,14 @@ processors:
         BASEPATH: '[[[:alnum:]]_%!$@:.,+~-]+'
         META: '\[[^\]]*\]'
   - date:
+      tag: date__tmp_timestamp8601_to_timestamp_6ac9d3ce
       if: ctx._tmp.timestamp8601 != null
       field: _tmp.timestamp8601
       target_field: '@timestamp'
       formats:
         - ISO8601
   - date:
+      tag: date__tmp_timestamp_to_timestamp_f21e536e
       if: ctx.event?.timezone != null && ctx._tmp?.timestamp != null
       field: _tmp.timestamp
       target_field: '@timestamp'
@@ -56,74 +65,93 @@ processors:
         - MMM dd HH:mm:ss
       timezone: '{{{ event.timezone }}}'
   - grok:
+      tag: grok_process_name_cef3d489
       description: Set Event Provider
       field: process.name
       patterns:
         - '^%{HYPHENATED_WORDS:event.provider}'
       pattern_definitions:
         HYPHENATED_WORDS: '\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\b'
   - pipeline:
+      tag: pipeline_e16851a7
       name: '{{ IngestPipeline "firewall" }}'
       if: ctx.event.provider == 'filterlog'
   - pipeline:
+      tag: pipeline_828590b5
       name: '{{ IngestPipeline "openvpn" }}'
       if: ctx.event.provider == 'openvpn'
   - pipeline:
+      tag: pipeline_9d37039c
       name: '{{ IngestPipeline "ipsec" }}'
       if: ctx.event.provider == 'charon'
   - pipeline:
+      tag: pipeline_ad56bbca
       name: '{{ IngestPipeline "dhcp" }}'
       if: '["dhcpd", "dhclient", "dhcp6c"].contains(ctx.event.provider)'
   - pipeline:
+      tag: pipeline_dd85553d
       name: '{{ IngestPipeline "unbound" }}'
       if: ctx.event.provider == 'unbound'
   - pipeline:
+      tag: pipeline_720ed255
       name: '{{ IngestPipeline "haproxy" }}'
       if: ctx.event.provider == 'haproxy'
   - pipeline:
+      tag: pipeline_456beba5
       name: '{{ IngestPipeline "php-fpm" }}'
       if: ctx.event.provider == 'php-fpm'
   - pipeline:
+      tag: pipeline_a0d89375
       name: '{{ IngestPipeline "squid" }}'
       if: ctx.event.provider == 'squid'
   - pipeline:
+      tag: pipeline_c2f1ed55
       name: '{{ IngestPipeline "snort" }}'
       if: ctx.event.provider == 'snort'
   - drop:
+      tag: drop_9d7c46f8
       if: '!["filterlog", "openvpn", "charon", "dhcpd", "dhclient", "dhcp6c", "unbound", "haproxy", "php-fpm", "squid", "snort"].contains(ctx.event?.provider)'
   - append:
+      tag: append_event_category_4780a983
       field: event.category
       value: network
       if: "ctx.network != null"
   - convert:
+      tag: convert_source_address_to_source_ip_f5632a20
       field: source.address
       target_field: source.ip
       type: ip
       ignore_failure: true
       ignore_missing: true
   - convert:
+      tag: convert_destination_address_to_destination_ip_f1388f0c
       field: destination.address
       target_field: destination.ip
       type: ip
       ignore_failure: true
       ignore_missing: true
   - set:
+      tag: set_network_type_1f1d940a
       field: network.type
       value: ipv6
       if: 'ctx.source?.ip != null && ctx.source.ip.contains(":")'
   - set:
+      tag: set_network_type_69deca38
       field: network.type
       value: ipv4
       if: 'ctx.source?.ip != null && ctx.source.ip.contains(".")'
   - geoip:
+      tag: geoip_source_ip_to_source_geo_da2e41b2
       field: source.ip
       target_field: source.geo
       ignore_missing: true
   - geoip:
+      tag: geoip_destination_ip_to_destination_geo_ab5e2968
       field: destination.ip
       target_field: destination.geo
       ignore_missing: true
   - geoip:
+      tag: geoip_source_ip_to_source_as_28d69883
       ignore_missing: true
       database_file: GeoLite2-ASN.mmdb
       field: source.ip
@@ -132,6 +160,7 @@ processors:
         - asn
         - organization_name
   - geoip:
+      tag: geoip_destination_ip_to_destination_as_8a007787
       database_file: GeoLite2-ASN.mmdb
       field: destination.ip
       target_field: destination.as
@@ -140,66 +169,81 @@ processors:
         - organization_name
       ignore_missing: true
   - rename:
+      tag: rename_source_as_asn_to_source_as_number_a917047d
       field: source.as.asn
       target_field: source.as.number
       ignore_missing: true
   - rename:
+      tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b
       field: source.as.organization_name
       target_field: source.as.organization.name
       ignore_missing: true
   - rename:
+      tag: rename_destination_as_asn_to_destination_as_number_3b459fcd
       field: destination.as.asn
       target_field: destination.as.number
       ignore_missing: true
   - rename:
+      tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
   - community_id:
+      tag: community_id_d2308e7a
       target_field: network.community_id
       ignore_failure: true
   - grok:
+      tag: grok_observer_ingress_interface_name_968018d3
       field: observer.ingress.interface.name
       patterns:
         - "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
       ignore_missing: true
       ignore_failure: true
   - set:
+      tag: set_network_vlan_id_efd4d96a
       field: network.vlan.id
       copy_from: observer.ingress.vlan.id
       ignore_empty_value: true
   - append:
+      tag: append_related_ip_c1a6356b
       field: related.ip
       value: "{{{destination.ip}}}"
       allow_duplicates: false
       if: ctx.destination?.ip != null
   - append:
+      tag: append_related_ip_8121c591
       field: related.ip
       value: "{{{source.ip}}}"
       allow_duplicates: false
       if: ctx.source?.ip != null
   - append:
+      tag: append_related_ip_53b62ed8
       field: related.ip
       value: "{{{source.nat.ip}}}"
       allow_duplicates: false
       if: ctx.source?.nat?.ip != null
   - append:
+      tag: append_related_hosts_6f162628
       field: related.hosts
       value: "{{{destination.domain}}}"
       if: "ctx.destination?.domain != null"
   - append:
+      tag: append_related_user_c036eec2
       field: related.user
       value: "{{{user.name}}}"
       if: "ctx.user?.name != null"
   - set:
+      tag: set_network_direction_cb1e3125
       field: network.direction
       value: "{{{network.direction}}}bound"
       if: ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/
   - remove:
+      tag: remove_a82e20f2
       field:
         - _tmp
       ignore_failure: true
   - script:
+      tag: script_a7f2c062
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.
       source: |
@@ -233,4 +277,8 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml

@@ -2,6 +2,7 @@
 description: Pipeline for PFsense DHCP logs
 processors:
   - grok:
+      tag: grok_message_efb70afb
       field: message
       patterns:
         - '%{DATA:_tmp.action}/%{INTERFACE:observer.ingress.interface.name}/%{MAC_ADDRESS:server.mac}/%{NOTSPACE:pfsense.dhcp.subnet}'
@@ -21,68 +22,83 @@ processors:
         MIDDLE: '(?:%{FROM}|%{ON}|%{FOR})'
         DUID: '(?i)[0-9a-f]{2}(:[0-9a-f]{2})+'
   - append:
+      tag: append_event_type_c741503d
       field: event.type
-      value: 
+      value:
         - connection
         - protocol
         - info
       allow_duplicates: false
   - set:
+      tag: set_network_protocol_ea539779
       field: network.protocol
       value: dhcp
   - set:
+      tag: set_network_protocol_98d42096
       field: network.protocol
       value: dhcpv6
       if: ctx.event.provider == 'dhcp6c' || (ctx.server?.address != null && ctx.server.address.contains(':')) || (ctx.client?.address != null && ctx.client.address.contains(':'))
   - set:
+      tag: set_network_transport_a6cfd568
       field: network.transport
       value: udp
   - convert:
+      tag: convert_client_address_to_client_ip_c718a334
       field: client.address
       target_field: client.ip
       type: ip
       ignore_failure: true
       ignore_missing: true
   - convert:
+      tag: convert_server_address_to_server_ip_940441bc
       field: server.address
       target_field: server.ip
       type: ip
       ignore_failure: true
       ignore_missing: true
   - uppercase:
+      tag: uppercase_client_mac_230df574
       field: client.mac
       ignore_missing: true
   - gsub:
+      tag: gsub_client_mac_3eb17338
       field: client.mac
       pattern: '[:]'
       replacement: '-'
       ignore_missing: true
   - uppercase:
+      tag: uppercase_server_mac_12d7a2a0
       field: server.mac
       ignore_missing: true
   - gsub:
+      tag: gsub_server_mac_5d413db4
       field: server.mac
       pattern: '[:]'
       replacement: '-'
       ignore_missing: true
   - lowercase:
+      tag: lowercase__tmp_action_5d4e19c7
       field: _tmp.action
       ignore_missing: true
   - gsub:
+      tag: gsub__tmp_action_to_event_action_2c25be60
       field: _tmp.action
       target_field: event.action
       pattern: ' '
       replacement: '-'
       ignore_missing: true
   - set:
+      tag: set_source_a194c11b
       field: source
       copy_from: client
       ignore_empty_value: true
   - set:
+      tag: set_destination_bb802684
       field: destination
       copy_from: server
       ignore_empty_value: true
   - append:
+      tag: append_related_hosts_8bba1f0b
       field: related.hosts
       value: "{{{pfsense.dhcp.hostname}}}"
       allow_duplicates: false
@@ -93,4 +109,8 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'