[system] Add support for more event-ids in the security data stream

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Add support for more event ids in the security data stream pipeline.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13828
 - version: "2.0.1"
   changes:
     - description: Fix missing `period` config in `core` data stream.

packages/system/data_stream/security/_dev/test/pipeline/test-1100.json-expected.json

@@ -34,6 +34,9 @@
                 },
                 "level": "information"
             },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
             "winlog": {
                 "channel": "Security",
                 "computer_name": "WIN-41OB2LO92CR.wlbeat.local",

packages/system/data_stream/security/_dev/test/pipeline/test-1102.json-expected.json

@@ -40,6 +40,9 @@
                     "Administrator"
                 ]
             },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
             "user": {
                 "domain": "WLBEAT",
                 "id": "S-1-5-21-101361758-2486510592-3018839910-500",

packages/system/data_stream/security/_dev/test/pipeline/test-1104.json-expected.json

@@ -34,6 +34,9 @@
                 },
                 "level": "error"
             },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
             "winlog": {
                 "channel": "Security",
                 "computer_name": "WIN-41OB2LO92CR.wlbeat.local",

packages/system/data_stream/security/_dev/test/pipeline/test-1105.json-expected.json

@@ -34,6 +34,9 @@
                 },
                 "level": "information"
             },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
             "winlog": {
                 "channel": "Security",
                 "computer_name": "WIN-41OB2LO92CR.wlbeat.local",

packages/system/data_stream/security/_dev/test/pipeline/test-4627.json

@@ -0,0 +1,64 @@
+{
+  "events": [
+      {
+          "@timestamp": "2023-01-05T12:53:42.125Z",
+          "agent": {
+            "type": "winlogbeat",
+            "ephemeral_id": "7af11293-39b6-4555-82f1-2653ee510182",
+            "hostname": "Server2",
+            "id": "e01362f3-fc45-4ba1-ac8a-52501b2c3abb",
+            "version": "7.3.1"
+          },
+          "winlog": {
+            "computer_name": "Server2.test1.local",
+            "event_data": {
+              "TargetDomainName": "TEST1.LOCAL",
+              "SubjectDomainName": "-",
+              "LogonType": "3",
+              "EventIdx": "1",
+              "EventCountTotal": "1",
+              "SubjectLogonId": "0x0",
+              "TargetUserSid": "S-1-5-18",
+              "TargetUserName": "SERVER2$",
+              "GroupMembership": "\n\t\t%{S-1-5-32-544}\n\t\t%{S-1-1-0}\n\t\t%{S-1-5-32-554}\n\t\t%{S-1-5-32-545}\n\t\t%{S-1-5-32-574}\n\t\t%{S-1-5-32-560}\n\t\t%{S-1-5-2}\n\t\t%{S-1-5-11}\n\t\t%{S-1-5-15}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-1000}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-516}\n\t\t%{S-1-5-9}\n\t\t%{S-1-18-1}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-517}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-571}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-572}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-553}\n\t\t%{S-1-16-16384}",
+              "SubjectUserName": "-",
+              "TargetLogonId": "0x7bf2cce",
+              "SubjectUserSid": "S-1-0-0"
+            },
+            "channel": "Security",
+            "provider_name": "Microsoft-Windows-Security-Auditing",
+            "record_id": 630235150,
+            "task": "Group Membership",
+            "event_id": 4627,
+            "api": "wineventlog",
+            "keywords": [
+              "Audit Success"
+            ],
+            "opcode": "Info",
+            "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+            "process": {
+              "pid": 652,
+              "thread": {
+                "id": 4192
+              }
+            }
+          },
+          "event": {
+            "kind": "event",
+            "code": 4627,
+            "action": "Group Membership",
+            "created": "2023-01-05T12:53:43.721Z"
+          },
+          "log": {
+            "level": "information"
+          },
+          "message": "Group membership information.\n\nSubject:\n\tSecurity ID:\t\tS-1-0-0\n\tAccount Name:\t\t-\n\tAccount Domain:\t\t-\n\tLogon ID:\t\t0x0\n\nLogon Type:\t\t\t3\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSERVER2$\n\tAccount Domain:\t\tTEST1.LOCAL\n\tLogon ID:\t\t0x7BF2CCE\n\nEvent in sequence:\t\t1 of 1\n\nGroup Membership:\t\t\t\n\t\t%{S-1-5-32-544}\n\t\t%{S-1-1-0}\n\t\t%{S-1-5-32-554}\n\t\t%{S-1-5-32-545}\n\t\t%{S-1-5-32-574}\n\t\t%{S-1-5-32-560}\n\t\t%{S-1-5-2}\n\t\t%{S-1-5-11}\n\t\t%{S-1-5-15}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-1000}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-516}\n\t\t%{S-1-5-9}\n\t\t%{S-1-18-1}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-517}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-571}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-572}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-553}\n\t\t%{S-1-16-16384}\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThis event is generated when the Audit Group Membership subcategory is configured.  The Logon ID field can be used to correlate this event with the corresponding user logon event as well as to any other security audit events generated during this logon session.",
+          "ecs": {
+            "version": "1.0.1"
+          },
+          "host": {
+            "name": "Server2"
+          }
+      }
+  ]
+}

packages/system/data_stream/security/_dev/test/pipeline/test-4627.json-expected.json

@@ -0,0 +1,99 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-01-05T12:53:42.125Z",
+            "agent": {
+                "ephemeral_id": "7af11293-39b6-4555-82f1-2653ee510182",
+                "hostname": "Server2",
+                "id": "e01362f3-fc45-4ba1-ac8a-52501b2c3abb",
+                "type": "winlogbeat",
+                "version": "7.3.1"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "group-membership-info",
+                "category": [
+                    "iam"
+                ],
+                "code": "4627",
+                "created": "2023-01-05T12:53:43.721Z",
+                "kind": "event",
+                "type": [
+                    "info"
+                ]
+            },
+            "group": {
+                "domain": "TEST1.LOCAL",
+                "id": "S-1-5-18",
+                "name": "SERVER2$"
+            },
+            "host": {
+                "name": "Server2"
+            },
+            "log": {
+                "level": "information"
+            },
+            "message": "Group membership information.\n\nSubject:\n\tSecurity ID:\t\tS-1-0-0\n\tAccount Name:\t\t-\n\tAccount Domain:\t\t-\n\tLogon ID:\t\t0x0\n\nLogon Type:\t\t\t3\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSERVER2$\n\tAccount Domain:\t\tTEST1.LOCAL\n\tLogon ID:\t\t0x7BF2CCE\n\nEvent in sequence:\t\t1 of 1\n\nGroup Membership:\t\t\t\n\t\t%{S-1-5-32-544}\n\t\t%{S-1-1-0}\n\t\t%{S-1-5-32-554}\n\t\t%{S-1-5-32-545}\n\t\t%{S-1-5-32-574}\n\t\t%{S-1-5-32-560}\n\t\t%{S-1-5-2}\n\t\t%{S-1-5-11}\n\t\t%{S-1-5-15}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-1000}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-516}\n\t\t%{S-1-5-9}\n\t\t%{S-1-18-1}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-517}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-571}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-572}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-553}\n\t\t%{S-1-16-16384}\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThis event is generated when the Audit Group Membership subcategory is configured.  The Logon ID field can be used to correlate this event with the corresponding user logon event as well as to any other security audit events generated during this logon session.",
+            "related": {
+                "user": [
+                    "SERVER2$"
+                ]
+            },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
+            "user": {
+                "domain": "TEST1.LOCAL",
+                "effective": {
+                    "domain": "TEST1.LOCAL",
+                    "id": "S-1-5-18",
+                    "name": "SERVER2$"
+                },
+                "id": "S-1-0-0",
+                "name": "SERVER2$",
+                "target": {
+                    "domain": "TEST1.LOCAL",
+                    "name": "SERVER2$"
+                }
+            },
+            "winlog": {
+                "api": "wineventlog",
+                "channel": "Security",
+                "computer_name": "Server2.test1.local",
+                "event_data": {
+                    "EventCountTotal": 1,
+                    "EventIdx": 1,
+                    "GroupMembership": "\n\t\t%{S-1-5-32-544}\n\t\t%{S-1-1-0}\n\t\t%{S-1-5-32-554}\n\t\t%{S-1-5-32-545}\n\t\t%{S-1-5-32-574}\n\t\t%{S-1-5-32-560}\n\t\t%{S-1-5-2}\n\t\t%{S-1-5-11}\n\t\t%{S-1-5-15}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-1000}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-516}\n\t\t%{S-1-5-9}\n\t\t%{S-1-18-1}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-517}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-571}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-572}\n\t\t%{S-1-5-21-1280187532-2219128962-763009249-553}\n\t\t%{S-1-16-16384}",
+                    "LogonType": "3",
+                    "SubjectLogonId": "0x0",
+                    "SubjectUserSid": "S-1-0-0",
+                    "TargetDomainName": "TEST1.LOCAL",
+                    "TargetLogonId": "0x7bf2cce",
+                    "TargetUserName": "SERVER2$",
+                    "TargetUserSid": "S-1-5-18"
+                },
+                "event_id": "4627",
+                "keywords": [
+                    "Audit Success"
+                ],
+                "logon": {
+                    "id": "0x0",
+                    "type": "Network"
+                },
+                "opcode": "Info",
+                "process": {
+                    "pid": 652,
+                    "thread": {
+                        "id": 4192
+                    }
+                },
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "record_id": "630235150",
+                "task": "Group Membership"
+            }
+        }
+    ]
+}

packages/system/data_stream/security/_dev/test/pipeline/test-4658.json

@@ -0,0 +1,60 @@
+{
+  "events": [
+    {
+      "@timestamp": "2023-01-06T05:50:32.050Z",
+      "host": {
+        "name": "Server2"
+      },
+      "agent": {
+        "version": "7.3.1",
+        "type": "winlogbeat",
+        "ephemeral_id": "7af11293-39b6-4555-82f1-2653ee510182",
+        "hostname": "Server2",
+        "id": "e01362f3-fc45-4ba1-ac8a-52501b2c3abb"
+      },
+      "log": {
+        "level": "information"
+      },
+      "message": "The handle to an object was closed.\n\nSubject :\n\tSecurity ID:\t\tS-1-5-20\n\tAccount Name:\t\tSERVER2$\n\tAccount Domain:\t\tTEST1\n\tLogon ID:\t\t0x3E4\n\nObject:\n\tObject Server:\t\tSecurity\n\tHandle ID:\t\t0x3b0\n\nProcess Information:\n\tProcess ID:\t\t0x92c\n\tProcess Name:\t\tC:\\Windows\\System32\\wbem\\WmiPrvSE.exe",
+      "winlog": {
+        "computer_name": "Server2.test1.local",
+        "opcode": "Info",
+        "channel": "Security",
+        "process": {
+          "pid": 4,
+          "thread": {
+            "id": 2716
+          }
+        },
+        "event_data": {
+          "SubjectLogonId": "0x3e4",
+          "ObjectServer": "Security",
+          "HandleId": "0x3b0",
+          "ProcessId": "0x92c",
+          "ProcessName": "C:\\Windows\\System32\\wbem\\WmiPrvSE.exe",
+          "SubjectUserSid": "S-1-5-20",
+          "SubjectUserName": "SERVER2$",
+          "SubjectDomainName": "TEST1"
+        },
+        "provider_name": "Microsoft-Windows-Security-Auditing",
+        "record_id": 633857954,
+        "task": "Removable Storage",
+        "api": "wineventlog",
+        "keywords": [
+          "Audit Success"
+        ],
+        "event_id": 4658,
+        "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}"
+      },
+      "event": {
+        "kind": "event",
+        "code": 4658,
+        "action": "Removable Storage",
+        "created": "2023-01-06T05:50:33.603Z"
+      },
+      "ecs": {
+        "version": "1.0.1"
+      }
+    }
+  ]
+}

packages/system/data_stream/security/_dev/test/pipeline/test-4658.json-expected.json

@@ -0,0 +1,86 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-01-06T05:50:32.050Z",
+            "agent": {
+                "ephemeral_id": "7af11293-39b6-4555-82f1-2653ee510182",
+                "hostname": "Server2",
+                "id": "e01362f3-fc45-4ba1-ac8a-52501b2c3abb",
+                "type": "winlogbeat",
+                "version": "7.3.1"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "handle-closed-object",
+                "category": [
+                    "iam",
+                    "configuration"
+                ],
+                "code": "4658",
+                "created": "2023-01-06T05:50:33.603Z",
+                "kind": "event",
+                "type": [
+                    "change"
+                ]
+            },
+            "host": {
+                "name": "Server2"
+            },
+            "log": {
+                "level": "information"
+            },
+            "message": "The handle to an object was closed.\n\nSubject :\n\tSecurity ID:\t\tS-1-5-20\n\tAccount Name:\t\tSERVER2$\n\tAccount Domain:\t\tTEST1\n\tLogon ID:\t\t0x3E4\n\nObject:\n\tObject Server:\t\tSecurity\n\tHandle ID:\t\t0x3b0\n\nProcess Information:\n\tProcess ID:\t\t0x92c\n\tProcess Name:\t\tC:\\Windows\\System32\\wbem\\WmiPrvSE.exe",
+            "process": {
+                "executable": "C:\\Windows\\System32\\wbem\\WmiPrvSE.exe",
+                "name": "WmiPrvSE.exe",
+                "pid": 2348
+            },
+            "related": {
+                "user": [
+                    "SERVER2$"
+                ]
+            },
+            "tags": [
+                "preserve_duplicate_custom_fields"
+            ],
+            "user": {
+                "domain": "TEST1",
+                "id": "S-1-5-20",
+                "name": "SERVER2$"
+            },
+            "winlog": {
+                "api": "wineventlog",
+                "channel": "Security",
+                "computer_name": "Server2.test1.local",
+                "event_data": {
+                    "HandleId": "0x3b0",
+                    "ObjectServer": "Security",
+                    "SubjectDomainName": "TEST1",
+                    "SubjectLogonId": "0x3e4",
+                    "SubjectUserName": "SERVER2$",
+                    "SubjectUserSid": "S-1-5-20"
+                },
+                "event_id": "4658",
+                "keywords": [
+                    "Audit Success"
+                ],
+                "logon": {
+                    "id": "0x3e4"
+                },
+                "opcode": "Info",
+                "process": {
+                    "pid": 4,
+                    "thread": {
+                        "id": 2716
+                    }
+                },
+                "provider_guid": "{54849625-5478-4994-A5BA-3E3B0328C30D}",
+                "provider_name": "Microsoft-Windows-Security-Auditing",
+                "record_id": "633857954",
+                "task": "Removable Storage"
+            }
+        }
+    ]
+}