Merge pull request SEKOIA-IO#970 from SEKOIA-IO/feat/improve-pa-ngfw

Palo Alto NGFW: Various improvements

Palo Alto Networks/paloalto-ngfw/_meta/smart-descriptions.json

@@ -139,6 +139,34 @@
       {
         "field": "log.logger",
         "value": "threat"
+      },
+      {
+        "field": "source.ip"
+      },
+      {
+        "field": "destination.ip"
+      }
+    ],
+    "relationships": [
+      {
+        "source": "source.ip",
+        "target": "destination.ip",
+        "type": "connected to"
+      }
+    ]
+  },
+  {
+    "value": "{event.action} threat between {source.nat.ip} and {destination.nat.ip}",
+    "conditions": [
+      {
+        "field": "log.logger",
+        "value": "threat"
+      },
+      {
+        "field": "source.nat.ip"
+      },
+      {
+        "field": "destination.nat.ip"
       }
     ],
     "relationships": [

Palo Alto Networks/paloalto-ngfw/ingest/parser.yml

@@ -320,7 +320,7 @@ pipeline:
           - URLID
           - UserAgent
           - FileType
-          - Xff
+          - xff
           - Referer
           - Sender
           - Subject
@@ -465,6 +465,7 @@ pipeline:
     filter: '{{parsed_event.message.get("EventDescription") != None}}'
 
   - name: set_extracted_fields
+  - name: set_finalize_user_name
   - name: set_category_fields
   - name: set_ecs_deviceOutboundInterface
     filter: '{{parsed_event.message.get("deviceOutboundInterface") != None}}'
@@ -507,7 +508,7 @@ stages:
           "@timestamp": "{{parsed_timestamp.datetime}}"
           event.start: "{{parsed_start.datetime}}"
           action.name: "{{parsed_event.message.Action or parsed_description.message.action}}"
-          action.type: "{{parsed_event.message.Subtype}}"
+          action.type: "{{parsed_event.message.Subtype|lower or parsed_event.message.Name|lower}}"
           container.id: "{{parsed_event.message.ContainerID}}"
           container.name: "{{parsed_event.message.ContainerName}}"
           destination.address: "{{parsed_event.message.DestinationAddress or parsed_event.message.dst or parsed_description.message.dst_addr}}"
@@ -555,13 +556,11 @@ stages:
           email.from.address: "{{parsed_event.message.Sender}}"
           email.subject: "{{parsed_event.message.Subject}}"
           email.to.address: "{{parsed_event.message.Recipient}}"
-          event.action: "{{parsed_event.message.act or parsed_event.message.Threat_Category or parsed_description.message.action}}"
+          event.action: "{{parsed_event.message.act or parsed_event.message.Action or parsed_description.message.action}}"
           event.timezone: "{{parsed_event.message.dtz}}"
           event.dataset: "{{parsed_event.message.DeviceEventClassID|lower or parsed_event.message.Type|lower or parsed_event.message.LogType|lower}}"
           event.reason: "{{parsed_event.message.reason or parsed_event.message.Threat_ContentName or parsed_event.message.EventDescription or parsed_event.message.PanOSConnectionError}}"
           event.module: "{{parsed_description.message.module}}"
-          file.path: "{{parsed_event.message.URLFilename}}"
-          file.name: "{{parsed_event.message.FileName or parsed_event.message.URLFilename or parsed_description.message.filename}}"
           host.hostname: "{{parsed_event.message.dvchost or parsed_event.message.PanOSEndpointDeviceName}}"
           host.name: "{{parsed_event.message.dvchost or parsed_event.message.PanOSEndpointDeviceName or parsed_event.message.LogSourceName or parsed_event.message.MachineName or parsed_event.message.shost}}"
           host.id: "{{parsed_event.message.deviceExternalId}}"
@@ -592,6 +591,23 @@ stages:
           rule.uuid: "{{parsed_event.message.PanOSRuleUUID or parsed_event.message.RuleUUID}}"
           source.bytes: "{{parsed_event.message.BytesSent or parsed_event.message.in}}"
 
+      - set:
+          file.path: "{{parsed_event.message.URLFilename}}"
+          file.name: "{{parsed_event.message.FileName or parsed_event.message.URLFilename or parsed_description.message.filename}}"
+        filter: "{{final.action.type != 'url'}}"
+
+      - set:
+          url.original: "{{parsed_event.message.FileName or parsed_event.message.URLFilename}}"
+          url.domain: '{{final.url.original.split("/")[0].split(":")[0]}}'
+          url.port: '{{final.url.original.split("/")[0].split(":")[1]}}'
+          url.path: '{{final.url.original.split("?")[0].split("/")[1:] | join("/")}}'
+          url.query: '{{final.url.original.split("?")[1]}}'
+        filter: "{{final.action.type == 'url'}}"
+
+      - delete:
+          - url.original
+        filter: "{{final.action.type == 'url'}}"
+
       - set:
           source.ip: "{{parsed_event.message.PublicIP}}"
         filter: "{{parsed_event.message.PublicIP | is_ipaddress}}"
@@ -639,6 +655,11 @@ stages:
       - set:
           source.nat.ip: "{{parsed_event.message.sourceTranslatedAddress}}"
         filter: "{{parsed_event.message.sourceTranslatedAddress | is_ipaddress}}"
+
+      - set:
+          network.forwarded_ip: "{{parsed_event.message.XFFAddress or parsed_event.message.xff}}"
+        filter: "{{parsed_event.message.XFFAddress | is_ipaddress or parsed_event.message.xff | is_ipaddress}}"
+
       - set:
           source.geo.country_iso_code: "{{parsed_event.message.SourceRegion or parsed_event.message.SourceLocation}}"
         filter: "{{parsed_event.message.SourceLocation | length == 2 or parsed_event.message.PanOSSourceLocation | length == 2}}"
@@ -653,11 +674,11 @@ stages:
           source.nat.port: "{{parsed_event.message.NATSourcePort or parsed_event.message.sourceTranslatedPort}}"
           source.packets: "{{parsed_event.message.PanOSPacketsSent or parsed_event.message.pkts_sent}}"
           source.port: "{{parsed_event.message.SourcePort or parsed_event.message.spt}}"
-          source.user.name: "{{parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_event.message.SourceUser}}"
+          source.user.name: "{{parsed_event.message.suser or parsed_event.message.PanOSSourceUserName}}"
           user_agent.name: "{{parsed_event.message.UserAgent}}"
           user_agent.os.name: "{{parsed_event.message.ClientOS}}"
           user_agent.os.version: "{{parsed_event.message.ClientOSVersion}}"
-          user.name: "{{parsed_event.message.SourceUser or parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}"
+          user.name: "{{parsed_event.message.User or parsed_event.message.suser or parsed_event.message.PanOSSourceUserName or parsed_description.message.user}}"
           paloalto: >-
             { 
             {% set ns = namespace(first_iteration=True) %}
@@ -675,6 +696,25 @@ stages:
           paloalto.connection.method: "{{parsed_event.message.ConnectionMethod or parsed_event.message.PanOSConnectionMethod}}"
           paloalto.endpoint.serial_number: "{{parsed_event.message.EndpointSerialNumber or parsed_event.message.PanOSEndpointSerialNumber}}"
           paloalto.threat.id: "{{parsed_event.message.ThreatID or parsed_event.message.PanOSThreatID}}"
+      - set:
+          source.user.name: "{{parsed_event.message.SourceUser}}"
+          user.name: "{{parsed_event.message.SourceUser}}"
+        filter: '{{parsed_event.message.SourceUser.startswith("x-fwd-for") == False}}'
+
+  set_finalize_user_name:
+    actions:
+      - set:
+          user.domain: '{{final.user.name.split("\\") | first}}'
+          user.name: '{{final.user.name.split("\\") | last}}'
+        filter: '{{final.user.name != null and "\\" in final.user.name}}'
+      - set:
+          source.user.domain: '{{final.source.user.name.split("\\") | first}}'
+          source.user.name: '{{final.source.user.name.split("\\") | last}}'
+        filter: '{{final.source.user.name != null and "\\" in final.source.user.name}}'
+      - set:
+          destination.user.domain: '{{final.destination.user.name.split("\\") | first}}'
+          destination.user.name: '{{final.destination.user.name.split("\\") | last}}'
+        filter: '{{final.destination.user.name != null and "\\" in final.destination.user.name}}'
 
   set_category_fields:
     actions:

Palo Alto Networks/paloalto-ngfw/tests/auth_cef.json

@@ -17,9 +17,13 @@
       ]
     },
     "@timestamp": "2021-02-28T18:20:54Z",
+    "action": {
+      "type": "radius"
+    },
     "destination": {
       "user": {
-        "name": "paloaltonetwork\\\\xxxxx"
+        "domain": "paloaltonetwork",
+        "name": "xxxxx"
       }
     },
     "host": {
@@ -65,7 +69,7 @@
         "xxxxx"
       ],
       "user": [
-        "paloaltonetwork\\\\xxxxx"
+        "xxxxx"
       ]
     }
   }

Palo Alto Networks/paloalto-ngfw/tests/decryption_cef.json

@@ -18,6 +18,9 @@
       ]
     },
     "@timestamp": "2021-03-01T20:35:54Z",
+    "action": {
+      "type": "end"
+    },
     "destination": {
       "address": "1.1.1.1",
       "ip": "1.1.1.1",
@@ -27,7 +30,8 @@
       },
       "port": 20122,
       "user": {
-        "name": "paloaltonetwork\\\\\\\\xxxxx"
+        "domain": "paloaltonetwork",
+        "name": "xxxxx"
       }
     },
     "log": {
@@ -65,7 +69,7 @@
         "1.1.1.1"
       ],
       "user": [
-        "paloaltonetwork\\\\\\\\xxxxx"
+        "xxxxx"
       ]
     },
     "rule": {
@@ -81,11 +85,13 @@
       },
       "port": 16524,
       "user": {
-        "name": "paloaltonetwork\\\\\\\\xxxxx"
+        "domain": "paloaltonetwork",
+        "name": "xxxxx"
       }
     },
     "user": {
-      "name": "paloaltonetwork\\\\\\\\xxxxx"
+      "domain": "paloaltonetwork",
+      "name": "xxxxx"
     }
   }
 }

Palo Alto Networks/paloalto-ngfw/tests/file_cef.json

@@ -18,6 +18,9 @@
       ]
     },
     "@timestamp": "2021-03-01T21:06:06Z",
+    "action": {
+      "type": "file"
+    },
     "destination": {
       "address": "1.1.1.1",
       "ip": "1.1.1.1",

Palo Alto Networks/paloalto-ngfw/tests/fix_bug_with_int.json

@@ -5,6 +5,7 @@
   "expected": {
     "message": "1,2023/06/16 10:41:44,001701003551,TRAFFIC,end,2305,2023/06/16 10:41:44,1.2.3.4,5.6.7.8,0.0.0.0,0.0.0.0,GEN_WINLOG_Users,domain\\pusername,userdest,windows-remote-management,vsys1,PDT_STD,INFRA_ADM,aaa.111,aaa.111,Syslog_Test,2023/06/16 10:41:44,234981,1,51413,5985,0,0,15,tcp,allow,2346,1974,372,9,90,16,30,0,69678105127,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,3,tcp-fin,0,0,0,0,,FWPA01,from-policy,,,0,,0,,N/A,0,0,0,0,5e7eca5b-f585-4633-bbd4-9ed431f7f95b,0,0,,,,,,,",
     "event": {
+      "action": "allow",
       "category": [
         "network"
       ],
@@ -63,7 +64,7 @@
         "5.6.7.8"
       ],
       "user": [
-        "domain\\pusername",
+        "pusername",
         "userdest"
       ]
     },
@@ -82,11 +83,13 @@
       "packets": 6,
       "port": 51413,
       "user": {
-        "name": "domain\\pusername"
+        "domain": "domain",
+        "name": "pusername"
       }
     },
     "user": {
-      "name": "domain\\pusername"
+      "domain": "domain",
+      "name": "pusername"
     }
   }
 }

Palo Alto Networks/paloalto-ngfw/tests/fix_bug_without_int.json

@@ -5,6 +5,7 @@
   "expected": {
     "message": "1,2023/06/16 10:41:44,001701003551,TRAFFIC,end,2305,2023/06/16 10:41:44,1.2.3.4,5.6.7.8,0.0.0.0,0.0.0.0,GEN_WINLOG_Users,domainusername,destuser,windows-remote-management,vsys1,PDT_STD,INFRA_ADM,aaa.111,aaa.111,Syslog_Test,2023/06/16 10:41:44,234981,1,51413,5985,0,0,0x1c,tcp,allow,2346,1974,372,9,2023/06/16 10:41:26,16,not-resolved,0,69678105127,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,3,tcp-fin,0,0,0,0,,FWPA01,from-policy,,,0,,0,,N/A,0,0,0,0,5e7eca5b-f585-4633-bbd4-9ed431f7f95b,0,0,,,,,,,",
     "event": {
+      "action": "allow",
       "category": [
         "network"
       ],

Palo Alto Networks/paloalto-ngfw/tests/globalprotect_cef.json

@@ -18,6 +18,10 @@
       ]
     },
     "@timestamp": "2021-03-01T20:35:54Z",
+    "action": {
+      "name": "satellite-gateway-update-route",
+      "type": "globalprotect"
+    },
     "host": {
       "hostname": "machine_name2",
       "name": "machine_name2",
@@ -52,16 +56,18 @@
         "machine_name2"
       ],
       "user": [
-        "xxxxx\\\\\\\\xxxxx"
+        "xxxxx"
       ]
     },
     "source": {
       "user": {
-        "name": "xxxxx\\\\\\\\xxxxx"
+        "domain": "xxxxx",
+        "name": "xxxxx"
       }
     },
     "user": {
-      "name": "xxxxx\\\\\\\\xxxxx"
+      "domain": "xxxxx",
+      "name": "xxxxx"
     }
   }
 }

Palo Alto Networks/paloalto-ngfw/tests/globalprotect_csv_2.json

@@ -46,7 +46,7 @@
         "88.120.236.74"
       ],
       "user": [
-        "example.org\\\\test"
+        "test"
       ]
     },
     "source": {
@@ -56,11 +56,13 @@
       },
       "ip": "88.120.236.74",
       "user": {
-        "name": "example.org\\\\test"
+        "domain": "example.org",
+        "name": "test"
       }
     },
     "user": {
-      "name": "example.org\\\\test"
+      "domain": "example.org",
+      "name": "test"
     },
     "user_agent": {
       "os": {

Palo Alto Networks/paloalto-ngfw/tests/icmp_allow_csv.json

@@ -5,6 +5,7 @@
   "expected": {
     "message": "<14>Sep 16 10:00:02 PP 1,9/16/19 10:00,1801017000,TRAFFIC,start,2049,9/16/19 10:00,1.2.3.4,4.3.2.1,1.2.3.4,10.0.1.2,PING,,,ping,vsys,AAAAA,Zone1,ethernet1/1,ae2.11,Secure,9/16/19 10:00,24100,3,0,0,0,0,0x500000,icmp,allow,222,222,0,3,9/16/19 10:00,0,any,0,50660388939,0x0,Spain,France,0,3,0,n/a,0,0,0,0,,PA,from-policy,,,0,,0,,N/A,0,0,0,0",
     "event": {
+      "action": "allow",
       "category": [
         "network"
       ],

Palo Alto Networks/paloalto-ngfw/tests/iptag_cef.json

@@ -17,6 +17,9 @@
       ]
     },
     "@timestamp": "2021-03-01T21:20:13Z",
+    "action": {
+      "type": "iptag"
+    },
     "destination": {
       "address": "1.1.1.1",
       "ip": "1.1.1.1"

Palo Alto Networks/paloalto-ngfw/tests/tcp_allow_csv.json

@@ -5,6 +5,7 @@
   "expected": {
     "message": "<14>Sep 16 10:00:02 PA-1 1,9/16/19 10:00,1801016000,TRAFFIC,start,2049,9/16/19 10:00,1.2.3.4,4.3.2.1,0.0.0.0,0.0.0.0,proxy1,,,web-browsing,vsys1234,v10213,zone1,a.1,b.2,Secure,9/16/19 10:00,60000,1,61000,80,0,0,0x0,tcp,allow,800,700,70,2,9/16/19 10:00,0,any,0,50660381839,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,2,1,n/a,0,0,0,0,,PP,from-policy,,,0,,0,,N/A,0,0,0,0",
     "event": {
+      "action": "allow",
       "category": [
         "network"
       ],

Palo Alto Networks/paloalto-ngfw/tests/test_file_alert_json.json

@@ -5,6 +5,7 @@
   "expected": {
     "message": "{\"TimeReceived\": \"2024-02-06T18:17:09.000000Z\", \"DeviceSN\": \"no-serial\", \"LogType\": \"THREAT\", \"Subtype\": \"file\", \"SubType\": \"file\", \"ConfigVersion\": \"10.2\", \"TimeGenerated\": \"2024-02-06T18:17:02.000000Z\", \"SourceAddress\": \"1.2.3.4\", \"DestinationAddress\": \"5.6.7.8\", \"NATSource\": \"9.10.11.12\", \"NATDestination\": \"5.6.7.8\", \"Rule\": \"Global_Outbound_internet_access\", \"SourceUser\": \"[email protected]\", \"DestinationUser\": null, \"Application\": \"web-browsing\", \"VirtualLocation\": \"vsys1\", \"FromZone\": \"trust\", \"ToZone\": \"untrust\", \"InboundInterface\": \"tunnel.1\", \"OutboundInterface\": \"ethernet1/1\", \"LogSetting\": \"default\", \"SessionID\": 1450762, \"RepeatCount\": 1, \"SourcePort\": 53514, \"DestinationPort\": 80, \"NATSourcePort\": 22444, \"NATDestinationPort\": 80, \"Protocol\": \"tcp\", \"Action\": \"alert\", \"FileName\": \"some_file_name\", \"URLCategory\": \"computer-and-internet-info\", \"VendorSeverity\": \"Low\", \"DirectionOfAttack\": \"server to client\", \"SequenceNo\": 7292474944208657622, \"SourceLocation\": \"Prisma-Mobile-Users-EMEA\", \"DestinationLocation\": \"US\", \"PacketID\": 0, \"FileHash\": null, \"ReportID\": 0, \"DGHierarchyLevel1\": 463, \"DGHierarchyLevel2\": 467, \"DGHierarchyLevel3\": 0, \"DGHierarchyLevel4\": 0, \"VirtualSystemName\": \"\", \"DeviceName\": \"GP cloud service\", \"SourceUUID\": null, \"DestinationUUID\": null, \"IMSI\": 0, \"IMEI\": null, \"ParentSessionID\": 0, \"ParentStartTime\": \"1970-01-01T00:00:00.000000Z\", \"Tunnel\": \"N/A\", \"ContentVersion\": \"577053022\", \"SigFlags\": 0, \"RuleUUID\": \"c38e111b-43fc-4de4-a17c-c372af557193\", \"HTTP2Connection\": 0, \"DynamicUserGroup\": null, \"X-Forwarded-ForIP\": null, \"SourceDeviceCategory\": null, \"SourceDeviceProfile\": null, \"SourceDeviceModel\": null, \"SourceDeviceVendor\": null, \"SourceDeviceOSFamily\": null, \"SourceDeviceOSVersion\": null, \"SourceDeviceHost\": null, \"SourceDeviceMac\": null, \"DestinationDeviceCategory\": null, \"DestinationDeviceProfile\": null, \"DestinationDeviceModel\": null, \"DestinationDeviceVendor\": null, \"DestinationDeviceOSFamily\": null, \"DestinationDeviceOSVersion\": null, \"DestinationDeviceHost\": null, \"DestinationDeviceMac\": null, \"ContainerID\": null, \"ContainerNameSpace\": null, \"ContainerName\": null, \"SourceEDL\": null, \"DestinationEDL\": null, \"HostID\": null, \"EndpointSerialNumber\": null, \"DomainEDL\": null, \"SourceDynamicAddressGroup\": null, \"DestinationDynamicAddressGroup\": null, \"PartialHash\": 0, \"TimeGeneratedHighResolution\": \"2024-02-06T18:17:02.077000Z\", \"ReasonForDataFilteringAction\": null, \"Justification\": null, \"NSSAINetworkSliceType\": null}",
     "event": {
+      "action": "alert",
       "category": [
         "file"
       ],

Palo Alto Networks/paloalto-ngfw/tests/test_globalprotect.json

@@ -49,7 +49,7 @@
         "1.2.3.4"
       ],
       "user": [
-        "test.fr\\JDOE"
+        "JDOE"
       ]
     },
     "source": {
@@ -59,11 +59,13 @@
       },
       "ip": "1.2.3.4",
       "user": {
-        "name": "test.fr\\JDOE"
+        "domain": "test.fr",
+        "name": "JDOE"
       }
     },
     "user": {
-      "name": "test.fr\\JDOE"
+      "domain": "test.fr",
+      "name": "JDOE"
     },
     "user_agent": {
       "os": {