Syslog decoders

decoder/common.go

@@ -0,0 +1,47 @@
+package decoder
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+func anyToInt(v any) (int, error) {
+	switch vNum := v.(type) {
+	case int:
+		return vNum, nil
+	case float64:
+		return int(vNum), nil
+	case json.Number:
+		vInt64, err := vNum.Int64()
+		if err != nil {
+			return 0, err
+		}
+		return int(vInt64), nil
+	default:
+		return 0, errors.New("value is not convertable to int")
+	}
+}
+
+// atoi is allocation free ASCII number to integer conversion
+func atoi(b []byte) (int, bool) {
+	if len(b) == 0 {
+		return 0, false
+	}
+	x := 0
+	for _, c := range b {
+		if c < '0' || '9' < c {
+			return 0, false
+		}
+		x = x*10 + int(c) - '0'
+	}
+	return x, true
+}
+
+func isDigit(c byte) bool {
+	return c >= '0' && c <= '9'
+}
+
+func checkNumber(num []byte, minimum, maximum int) bool {
+	x, ok := atoi(num)
+	return ok && x >= minimum && x <= maximum
+}

decoder/decoder.go

@@ -13,6 +13,8 @@ const (
 	POSTGRES
 	NGINX_ERROR
 	PROTOBUF
+	SYSLOG_RFC3164
+	SYSLOG_RFC5424
 )
 
 type Decoder interface {

decoder/json.go

@@ -1,7 +1,6 @@
 package decoder
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"slices"
@@ -53,13 +52,13 @@ func (d *jsonDecoder) Type() Type {
 	return JSON
 }
 
-// DecodeToJson decodes json-formatted string and merges result with root.
+// DecodeToJson decodes json formatted string and merges result with root.
 func (d *jsonDecoder) DecodeToJson(root *insaneJSON.Root, data []byte) error {
 	data = d.cutFieldsBySize(data)
 	return root.DecodeBytes(data)
 }
 
-// Decode decodes json-formatted string to [*insaneJSON.Node].
+// Decode decodes json formatted string to [*insaneJSON.Node].
 //
 // Args:
 //   - root [*insaneJSON.Root] - required
@@ -141,23 +140,10 @@ func extractJsonParams(params map[string]any) (jsonParams, error) {
 			return jsonParams{}, fmt.Errorf("%q must be map", jsonMaxFieldsSizeParam)
 		}
 		for k, v := range maxFieldsSizeMap {
-			var vInt int
-
-			switch vNum := v.(type) {
-			case int:
-				vInt = vNum
-			case float64:
-				vInt = int(vNum)
-			case json.Number:
-				vInt64, err := vNum.Int64()
-				if err != nil {
-					return jsonParams{}, fmt.Errorf("each value in %q must be int", jsonMaxFieldsSizeParam)
-				}
-				vInt = int(vInt64)
-			default:
+			vInt, err := anyToInt(v)
+			if err != nil {
 				return jsonParams{}, fmt.Errorf("each value in %q must be int", jsonMaxFieldsSizeParam)
 			}
-
 			maxFieldsSize[k] = vInt
 		}
 	}

decoder/nginx.go

@@ -46,7 +46,7 @@ func (d *nginxErrorDecoder) Type() Type {
 	return NGINX_ERROR
 }
 
-// DecodeToJson decodes nginx error formatted log and merges result with root.
+// DecodeToJson decodes nginx-error formatted log and merges result with root.
 //
 // From:
 //
@@ -86,7 +86,7 @@ func (d *nginxErrorDecoder) DecodeToJson(root *insaneJSON.Root, data []byte) err
 	return nil
 }
 
-// Decode decodes nginx error formated log to [NginxErrorRow].
+// Decode decodes nginx-error formatted log to [NginxErrorRow].
 //
 // Example of format:
 //

decoder/postgres.go

@@ -25,7 +25,7 @@ type PostgresRow struct {
 	Log              []byte
 }
 
-// DecodePostgresToJson decodes postgres formatted log and merges result with event.
+// DecodePostgresToJson decodes postgres formatted log and merges result with root.
 //
 // From:
 //
@@ -42,24 +42,24 @@ type PostgresRow struct {
 //		"user": "test_user",
 //		"log": "listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\""
 //	}
-func DecodePostgresToJson(event *insaneJSON.Root, data []byte) error {
+func DecodePostgresToJson(root *insaneJSON.Root, data []byte) error {
 	row, err := DecodePostgres(data)
 	if err != nil {
 		return err
 	}
 
-	event.AddFieldNoAlloc(event, "time").MutateToBytesCopy(event, row.Time)
-	event.AddFieldNoAlloc(event, "pid").MutateToBytesCopy(event, row.PID)
-	event.AddFieldNoAlloc(event, "pid_message_number").MutateToBytesCopy(event, row.PIDMessageNumber)
-	event.AddFieldNoAlloc(event, "client").MutateToBytesCopy(event, row.Client)
-	event.AddFieldNoAlloc(event, "db").MutateToBytesCopy(event, row.DB)
-	event.AddFieldNoAlloc(event, "user").MutateToBytesCopy(event, row.User)
-	event.AddFieldNoAlloc(event, "log").MutateToBytesCopy(event, row.Log)
+	root.AddFieldNoAlloc(root, "time").MutateToBytesCopy(root, row.Time)
+	root.AddFieldNoAlloc(root, "pid").MutateToBytesCopy(root, row.PID)
+	root.AddFieldNoAlloc(root, "pid_message_number").MutateToBytesCopy(root, row.PIDMessageNumber)
+	root.AddFieldNoAlloc(root, "client").MutateToBytesCopy(root, row.Client)
+	root.AddFieldNoAlloc(root, "db").MutateToBytesCopy(root, row.DB)
+	root.AddFieldNoAlloc(root, "user").MutateToBytesCopy(root, row.User)
+	root.AddFieldNoAlloc(root, "log").MutateToBytesCopy(root, row.Log)
 
 	return nil
 }
 
-// DecodePostgres decodes postgres formated log to [PostgresRow].
+// DecodePostgres decodes postgres formatted log to [PostgresRow].
 //
 // Example of format:
 //

decoder/readme.md

@@ -12,7 +12,9 @@ Available values for `decoder` param:
 + cri -- parses cri format from log into event (e.g. `2016-10-06T00:17:09.669794203Z stderr F log content`)
 + postgres -- parses postgres format from log into event (e.g. `2021-06-22 16:24:27 GMT [7291] => [3-1] client=test_client,db=test_db,user=test_user LOG:  listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\"\n`)
 + nginx_error -- parses nginx error log format from log into event (e.g. `2022/08/17 10:49:27 [error] 2725122#2725122: *792412315 lua udp socket read timed out, context: ngx.timer`)
-+ protobuf -- parses protobuf message into event 
++ protobuf -- parses protobuf message into event
++ syslog_rfc3164 -- parses syslog-RFC3164 format from log into event (see [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164))
++ syslog_rfc5424 -- parses syslog-RFC5424 format from log into event (see [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424))
 
 > Currently `auto` is available only for usage with k8s input plugin.
 
@@ -71,7 +73,14 @@ To:
 }
 ```
 
-## Nginx decoder
+## Nginx-error decoder
+The resulting event may contain any of the following fields:
+* `time` *string*
+* `level` *string*
+* `pid` *string*
+* `tid` *string*
+* `cid` *string*
+* `message` *string*
 
 ### Params
 * `nginx_with_custom_fields` - if set, custom fields will be extracted.
@@ -86,7 +95,7 @@ pipelines:
 ```
 From:
 
-`"2022/08/17 10:49:27 [error] 2725122#2725122: *792412315 lua udp socket read timed out, context: ngx.timer"`
+`2022/08/17 10:49:27 [error] 2725122#2725122: *792412315 lua udp socket read timed out, context: ngx.timer`
 
 To:
 ```json
@@ -212,4 +221,158 @@ pipelines:
         proto_import_paths:
           - path/to/proto_dir1
           - path/to/proto_dir2
+```
+
+## Syslog-RFC3164 decoder
+The resulting event may contain any of the following fields:
+* `priority` *string*
+* `facility` *string*
+* `severity` *string*
+* `timestamp` *string* (`Stamp` format)
+* `hostname` *string*
+* `app_name` *string*
+* `process_id` *string*
+* `message` *string*
+
+### Params
+* `syslog_facility_format` - facility format, must be one of `number|string` (`number` by default).
+* `syslog_severity_format` - severity format, must be one of `number|string` (`number` by default).
+
+### Examples
+Default decoder:
+```yml
+pipelines:
+  example:
+    settings:
+      decoder: 'syslog_rfc3164'
+```
+From:
+
+`<34>Oct  5 22:14:15 mymachine.example.com myproc[10]: 'myproc' failed on /dev/pts/8`
+
+To:
+```json
+{
+  "priority": "34",
+  "facility": "4",
+  "severity": "2",
+  "timestamp": "Oct  5 22:14:15",
+  "hostname": "mymachine.example.com",
+  "app_name": "myproc",
+  "process_id": "10",
+  "message": "'myproc' failed on /dev/pts/8"
+}
+```
+---
+Decoder with `syslog_*_format` params:
+```yaml
+pipelines:
+  example:
+    settings:
+      decoder: 'syslog_rfc3164'
+      decoder_params:
+        syslog_facility_format: 'string'
+        syslog_severity_format: 'string'
+```
+From:
+
+`<34>Oct 11 22:14:15 mymachine.example.com myproc: 'myproc' failed on /dev/pts/8`
+
+To:
+```json
+{
+  "priority": "34",
+  "facility": "AUTH",
+  "severity": "CRIT",
+  "timestamp": "Oct 11 22:14:15",
+  "hostname": "mymachine.example.com",
+  "app_name": "myproc",
+  "message": "'myproc' failed on /dev/pts/8"
+}
+```
+
+## Syslog-RFC5424 decoder
+The resulting event may contain any of the following fields:
+* `priority` *string*
+* `facility` *string*
+* `severity` *string*
+* `proto_version` *string*
+* `timestamp` *string* (`RFC3339`/`RFC3339Nano` format)
+* `hostname` *string*
+* `app_name` *string*
+* `process_id` *string*
+* `message_id` *string*
+* `message` *string*
+* `SD_1` *object*
+* ...
+* `SD_N` *object*
+
+### Params
+* `syslog_facility_format` - facility format, must be one of `number|string` (`number` by default).
+* `syslog_severity_format` - severity format, must be one of `number|string` (`number` by default).
+
+### Examples
+Default decoder:
+```yml
+pipelines:
+  example:
+    settings:
+      decoder: 'syslog_rfc5424'
+```
+From:
+
+`<165>1 2003-10-11T22:14:15.003Z mymachine.example.com myproc 10 ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] An application event log`
+
+To:
+```json
+{
+  "priority": "165",
+  "facility": "20",
+  "severity": "5",
+  "proto_version": "1",
+  "timestamp": "2003-10-11T22:14:15.003Z",
+  "hostname": "mymachine.example.com",
+  "app_name": "myproc",
+  "process_id": "10",
+  "message_id": "ID47",
+  "message": "An application event log",
+  "exampleSDID@32473": {
+    "iut": "3",
+    "eventSource": "Application",
+    "eventID": "1011"
+  }
+}
+```
+---
+Decoder with `syslog_*_format` params:
+```yaml
+pipelines:
+  example:
+    settings:
+      decoder: 'syslog_rfc5424'
+      decoder_params:
+        syslog_facility_format: 'string'
+        syslog_severity_format: 'string'
+```
+From:
+
+`<165>1 2003-10-11T22:14:15.003Z mymachine.example.com myproc - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"]`
+
+To:
+```json
+{
+  "priority": "165",
+  "facility": "LOCAL4",
+  "severity": "NOTICE",
+  "proto_version": "1",
+  "timestamp": "2003-10-11T22:14:15.003Z",
+  "hostname": "mymachine.example.com",
+  "app_name": "myproc",
+  "message_id": "ID47",
+  "exampleSDID@32473": {
+    "iut": "3",
+    "eventSource": "Application",
+    "eventID": "1011"
+  }
+}
 ```