Skip to content

Commit 83291f8

Browse files
Kavindu-DodanKavindu-Dodan
committed
chore: otel to ecf convertor pipeline
Signed-off-by: Kavindu Dodanduwa <[email protected]> # Conflicts: # packages/aws/changelog.yml # packages/aws/manifest.yml
1 parent 0aee7a6 commit 83291f8

File tree

13 files changed

+176
-11
lines changed

13 files changed

+176
-11
lines changed

‎packages/aws/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "4.4.0"
3+
changes:
4+
- description: Add VPC flow log compatibility for OTel shim pipeline.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/99999
27
- version: "4.3.0"
38
changes:
49
- description: Improve documentation to align with new guidelines.

‎packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,50 +25,51 @@ processors:
2525
field: event.category
2626
value: [network]
2727
- drop:
28-
if: 'ctx.event?.original.startsWith("version") || ctx.event?.original.startsWith("instance-id")'
28+
if: ctx?.event?.source != 'otel' && (ctx.event?.original?.startsWith("version") || ctx.event?.original?.startsWith("instance-id"))
2929
- dissect:
3030
field: event.original
3131
pattern: '{"message":"%{event.original}"}'
32+
if: ctx?.event?.source != 'otel'
3233
ignore_failure: true
3334
- script:
3435
lang: painless
35-
if: ctx.event?.original != null
36+
if: ctx.event?.original != null && ctx?.event?.source != 'otel'
3637
source: >-
3738
ctx._temp_ = new HashMap();
3839
ctx._temp_.message_token_count = ctx.event?.original.splitOnToken(" ").length;
3940
- dissect:
4041
field: event.original
4142
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status}'
42-
if: ctx?._temp_?.message_token_count == 14
43+
if: ctx?._temp_?.message_token_count == 14 && ctx?.event?.source != 'otel'
4344
- dissect:
4445
field: event.original
4546
pattern: '%{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr}'
46-
if: ctx?._temp_?.message_token_count == 6
47+
if: ctx?._temp_?.message_token_count == 6 && ctx?.event?.source != 'otel'
4748
- dissect:
4849
field: event.original
4950
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.action} %{aws.vpcflow.log_status}'
50-
if: ctx?._temp_?.message_token_count == 17
51+
if: ctx?._temp_?.message_token_count == 17 && ctx?.event?.source != 'otel'
5152
- dissect:
5253
field: event.original
5354
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.type} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.protocol} %{aws.vpcflow.bytes} %{aws.vpcflow.packets} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.log_status}'
54-
if: ctx?._temp_?.message_token_count == 21
55+
if: ctx?._temp_?.message_token_count == 21 && ctx?.event?.source != 'otel'
5556
- dissect:
5657
field: event.original
5758
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path}'
58-
if: ctx?._temp_?.message_token_count == 29
59+
if: ctx?._temp_?.message_token_count == 29 && ctx?.event?.source != 'otel'
5960
- dissect:
6061
field: event.original
6162
description: default format for transit gateway vpc flow logs, covering versions v2 through v6.
6263
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.resource_type} %{aws.vpcflow.account_id} %{aws.vpcflow.tgw_id} %{aws.vpcflow.tgw_attachment_id} %{aws.vpcflow.tgw_src_vpc_account_id} %{aws.vpcflow.tgw_dst_vpc_account_id} %{aws.vpcflow.tgw_src_vpc_id} %{aws.vpcflow.tgw_dst_vpc_id} %{aws.vpcflow.tgw_src_subnet_id} %{aws.vpcflow.tgw_dst_subnet_id} %{aws.vpcflow.tgw_src_eni} %{aws.vpcflow.tgw_dst_eni} %{aws.vpcflow.tgw_src_az_id} %{aws.vpcflow.tgw_dst_az_id} %{aws.vpcflow.tgw_pair_attachment_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.log_status} %{aws.vpcflow.type} %{aws.vpcflow.packets_lost_no_route} %{aws.vpcflow.packets_lost_blackhole} %{aws.vpcflow.packets_lost_mtu_exceeded} %{aws.vpcflow.packets_lost_ttl_expired} %{aws.vpcflow.tcp_flags} %{cloud.region} %{network.direction} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service}'
63-
if: ctx?._temp_?.message_token_count == 36
64+
if: ctx?._temp_?.message_token_count == 36 && ctx?.event?.source != 'otel'
6465
- dissect:
6566
field: event.original
6667
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path} %{aws.vpcflow.ecs_cluster_arn} %{aws.vpcflow.ecs_cluster_name} %{aws.vpcflow.ecs_container_instance_arn} %{aws.vpcflow.ecs_container_instance_id} %{aws.vpcflow.ecs_container_id} %{aws.vpcflow.ecs_second_container_id} %{aws.vpcflow.ecs_service_name} %{aws.vpcflow.ecs_task_definition_arn} %{aws.vpcflow.ecs_task_arn} %{aws.vpcflow.ecs_task_id}'
67-
if: ctx?._temp_?.message_token_count == 39
68+
if: ctx?._temp_?.message_token_count == 39 && ctx?.event?.source != 'otel'
6869
- dissect:
6970
field: event.original
7071
pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path} %{aws.vpcflow.ecs_cluster_arn} %{aws.vpcflow.ecs_cluster_name} %{aws.vpcflow.ecs_container_instance_arn} %{aws.vpcflow.ecs_container_instance_id} %{aws.vpcflow.ecs_container_id} %{aws.vpcflow.ecs_second_container_id} %{aws.vpcflow.ecs_service_name} %{aws.vpcflow.ecs_task_definition_arn} %{aws.vpcflow.ecs_task_arn} %{aws.vpcflow.ecs_task_id} %{aws.vpcflow.reject_reason}'
71-
if: ctx?._temp_?.message_token_count == 40
72+
if: ctx?._temp_?.message_token_count == 40 && ctx?.event?.source != 'otel'
7273

7374
# Convert Unix epoch to timestamp
7475
- date:
@@ -77,18 +78,21 @@ processors:
7778
ignore_failure: true
7879
formats:
7980
- UNIX
81+
if: ctx?.event?.source != 'otel'
8082
- date:
8183
field: aws.vpcflow.start
8284
target_field: event.start
8385
ignore_failure: true
8486
formats:
8587
- UNIX
88+
if: ctx?.event?.source != 'otel'
8689
- date:
8790
field: aws.vpcflow.end
8891
target_field: event.end
8992
ignore_failure: true
9093
formats:
9194
- UNIX
95+
if: ctx?.event?.source != 'otel'
9296
- remove:
9397
field:
9498
- aws.vpcflow.start

‎packages/aws/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 3.3.2
22
name: aws
33
title: AWS
4-
version: "4.3.0"
4+
version: 4.4.0
55
description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
66
type: integration
77
categories:

‎packages/otel_shim/_dev/build/build.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
dependencies:
2+
ecs:
3+
reference: [email protected]
4+
import_mappings: true

‎packages/otel_shim/_dev/build/docs/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# OTel shims pipelines
2+
3+
TODO
4+
5+
## Data streams
6+
7+
TODO
8+
## Requirements
9+
10+
You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
11+
You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
12+
13+
## Setup
14+
15+
TODO

‎packages/otel_shim/changelog.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# newer versions go on top
2+
- version: "0.1.1"
3+
changes:
4+
- description: Initial draft of the package
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

‎packages/otel_shim/data_stream/aws_vpcflow/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
---
2+
description: Pipeline for processing sample logs
3+
processors:
4+
- dot_expander:
5+
field: "*"
6+
path: attributes
7+
- dot_expander:
8+
field: "*"
9+
path: resource.attributes
10+
- set:
11+
field: aws.vpcflow.version
12+
copy_from: attributes.aws.vpc.flow.log.version
13+
- set:
14+
field: aws.vpcflow.account_id
15+
copy_from: resource.attributes.cloud.account.id
16+
- set:
17+
field: aws.vpcflow.interface_id
18+
copy_from: attributes.network.interface.name
19+
- set:
20+
field: aws.vpcflow.srcaddr
21+
copy_from: attributes.source.address
22+
- set:
23+
field: aws.vpcflow.dstaddr
24+
copy_from: attributes.destination.address
25+
- set:
26+
field: aws.vpcflow.srcport
27+
copy_from: attributes.source.port
28+
- set:
29+
field: aws.vpcflow.dstport
30+
copy_from: attributes.destination.port
31+
- set:
32+
field: aws.vpcflow.protocol
33+
copy_from: attributes.network.protocol.name
34+
- set:
35+
field: aws.vpcflow.packets
36+
copy_from: attributes.aws.vpc.flow.packets
37+
- set:
38+
field: aws.vpcflow.bytes
39+
copy_from: attributes.aws.vpc.flow.bytes
40+
- script:
41+
lang: painless
42+
source: |
43+
ctx.tmpStart = Instant.ofEpochMilli(ctx.attributes.aws.vpc.flow.start).toEpochMilli();
44+
- date:
45+
field: tmpStart
46+
target_field: event.start
47+
ignore_failure: true
48+
formats:
49+
- UNIX
50+
- set:
51+
field: event.end
52+
copy_from: '@timestamp'
53+
- set:
54+
field: aws.vpcflow.action
55+
copy_from: attributes.aws.vpc.flow.action
56+
- set:
57+
field: aws.vpcflow.log_status
58+
copy_from: attributes.aws.vpc.flow.status
59+
- remove:
60+
field:
61+
- attributes
62+
ignore_missing: true
63+
- set:
64+
field: event.source
65+
value: otel
66+
on_failure:
67+
- set:
68+
field: error.message
69+
value: '{{ _ingest.on_failure_message }}'

‎packages/otel_shim/data_stream/aws_vpcflow/fields/base-fields.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
- name: data_stream.type
2+
type: constant_keyword
3+
description: Data stream type.
4+
- name: data_stream.dataset
5+
type: constant_keyword
6+
description: Data stream dataset.
7+
- name: data_stream.namespace
8+
type: constant_keyword
9+
description: Data stream namespace.
10+
- name: '@timestamp'
11+
type: date
12+
description: Event timestamp.

‎packages/otel_shim/data_stream/aws_vpcflow/manifest.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
title: "Route OTel AWS VPC Flow Logs to ECS"
2+
type: logs
3+
dataset: aws.vpcflow.otel
4+
elasticsearch:
5+
dynamic_dataset: true
6+
dynamic_namespace: true

‎packages/otel_shim/data_stream/aws_vpcflow/routing_rules.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
- source_dataset: aws.vpcflow.otel
2+
rules:
3+
# Route to aws.vpcflow dataset if event parsing is successful
4+
- target_dataset: aws.vpcflow
5+
if: ctx?.event?.source == "otel"
6+
namespace:
7+
- "{{data_stream.namespace}}"
8+
- default

0 commit comments

Comments
 (0)