Merge pull request SEKOIA-IO#1096 from SEKOIA-IO/feat/parse_more_open…

…ssh_events

Parse more OpenSSH events

OpenSSH/openssh/ingest/parser.yml

@@ -10,13 +10,16 @@ pipeline:
           SSHD_HEADER: (%{MONTH:month} %{MONTHDAY:month_day} %{TIME:time} %{HOSTNAME:hostname} sshd\[%{INT}\]:)?
           SSHD_TEXT: ((%{WORD}|,)%{SPACE})*
           SSHD_TWO_WORDS: "%{WORD}%{SPACE}%{WORD}"
-          SSHD_MESSAGE_ACCEPTED: Accepted
+          SSHD_MESSAGE_ACCEPTED_POSTPONED: Accepted|Postponed
           SSHD_MESSAGE_CONNECTION_CLOSED: Connection closed by
+          SSHD_MESSAGE_CONNECTION_FROM: Connection from
           SSHD_MESSAGE_DISCONNECTED: Disconnected from
           SSHD_MESSAGE_RECEIVED_DISCONNECT: Received disconnect from
           SSHD_MESSAGE_CONNECTION_RESET: Connection reset by
+          SSHD_MESSAGE_SESSION_STARTING: Starting session
           SSHD_MESSAGE_PAM_SESSION_CLOSED: 'pam_unix\(sshd:session\): session closed for user'
           SSHD_MESSAGE_PAM_SESSION_OPENED: 'pam_unix\(sshd:session\): session opened for user'
+          SSHD_MESSAGE_PAM_FAILLOCK: 'pam_faillock\(sshd:auth\)'
           SSHD_MESSAGE_PAM: 'PAM service\(sshd\)'
           SSHD_MESSAGE_BAD_PROTOCOL: Bad protocol version
           SSHD_MESSAGE_IDENTIFICATION_NOT_RECEIVED: Did not receive identification string
@@ -29,6 +32,9 @@ pipeline:
           SSHD_MESSAGE_PAM_AUTH_FAILURE: 'pam_unix\(sshd:auth\): authentication failure;'
           SSHD_MESSAGE_PAM_EXPIRED_PASSWORD: 'pam_unix\(sshd:account\): expired password'
           SSHD_MESSAGE_PAM_WIN_GRANTED_ACCESS: 'pam_winbind\(sshd:account\)'
+          SSHD_MESSAGE_PAMD_FAILED: 'pam_systemd\(sshd:session\): Failed to create session:'
+          SSHD_MESSAGE_PAM_FAULTY_MODULE: "PAM adding faulty module:"
+          SSHD_MESSAGE_PAM_UNABLE_DLOPEN: "PAM unable to dlopen"
           SSHD_MESSAGE_FAILED_PASSWORD: "Failed password|Failed none"
           SSHD_MESSAGE_INVALID_USER2: Invalid
           SSHD_MESSAGE_USER2: User
@@ -40,22 +46,25 @@ pipeline:
           SSHD_MESSAGE_USER: "(%{SSHD_MESSAGE_USER_QUALITY:userquality})|((%{SSHD_MESSAGE_USER_QUALITY:userquality} )?(%{USERNAME:user_name}|[\\*]+)(@%{HOSTNAME:source_domain})?)" # ***** for redacted
           SSHD_CLIENT_ADDRESS: "%{IP:source_ip}|%{HOSTNAME:source_domain}"
           SSHD_INFO_ACCEPTED:
-            "%{SSHD_MESSAGE_ACCEPTED:sshd_message} %{NOTSPACE:outcome_reason}
+            "%{SSHD_MESSAGE_ACCEPTED_POSTPONED:sshd_message} %{NOTSPACE:outcome_reason}
             for %{USERNAME:user_name}(@%{HOSTNAME:source_domain})?( from %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port}(
             ssh2: %{NOTSPACE} SHA256:%{NOTSPACE:file_hash_sha256}|%{GREEDYDATA}))?"
+          SSHD_INFO_KEY_ACCEPTED: '%{SSHD_MESSAGE_ACCEPTED_POSTPONED:sshd_message} key %{DATA} %{DATA:public_key_hash} found at %{DATA:key_path}\:%{NUMBER}'
           SSHD_INFO_CONNECTION_CLOSED:
             '%{SSHD_MESSAGE_CONNECTION_CLOSED:sshd_message}(%{SPACE}%{SSHD_MESSAGE_USER})?
-            %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port}( \[preauth\])?'
+            %{SSHD_CLIENT_ADDRESS}( port %{NUMBER:source_port}( \[preauth\])?)?'
+          SSHD_INFO_CONNECTION: "%{SSHD_MESSAGE_CONNECTION_FROM:sshd_message} %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port} on %{IP:destination_ip} port %{NUMBER:destination_port}"
           SSHD_INFO_DISCONNECTED:
             '%{SSHD_MESSAGE_DISCONNECTED:sshd_message}(%{SPACE}%{SSHD_MESSAGE_USER})?
             %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port}( \[preauth\])?'
-          SSHD_INFO_RECEIVED_DISCONNECT:
-            '%{SSHD_MESSAGE_RECEIVED_DISCONNECT:sshd_message} %{SSHD_CLIENT_ADDRESS}
-            port %{NUMBER:source_port}:%{INT}:%{SPACE}%{SSHD_TEXT:outcome_reason}( \[preauth\])?'
+          SSHD_INFO_RECEIVED_DISCONNECT: '%{SSHD_MESSAGE_RECEIVED_DISCONNECT:sshd_message} %{SSHD_CLIENT_ADDRESS}( port %{NUMBER:source_port})?:%{SPACE}?%{INT}:%{SPACE}%{SSHD_TEXT:outcome_reason}( \[preauth\])?'
           SSHD_INFO_CONNECTION_RESET:
             '%{SSHD_MESSAGE_CONNECTION_RESET:sshd_message}( %{SSHD_MESSAGE_USER})?
             %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port}( \[preauth\])?'
           SSHD_INFO_DISCONNECTION: "%{SSHD_INFO_CONNECTION_CLOSED}|%{SSHD_INFO_DISCONNECTED}|%{SSHD_INFO_RECEIVED_DISCONNECT}|%{SSHD_INFO_CONNECTION_RESET}"
+          SSHD_INFO_SESSION_STARTING: "%{SSHD_MESSAGE_SESSION_STARTING:sshd_message}: command for %{SSHD_MESSAGE_USER} from %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port} id %{NUMBER}"
+          SSHD_INFO_USER_ON_PID: "%{SSHD_MESSAGE_USER2:sshd_message} %{SSHD_MESSAGE_USER} (?P<outcome_reason>is on pid %{NUMBER:pid})"
+          SSHD_INFO_RSA_DISABLED: "(?P<sshd_message>main: sshd: ssh-rsa algorithm is disabled)"
           SSHD_INFO_PAM_SESSION_CLOSED: "%{SSHD_MESSAGE_PAM_SESSION_CLOSED:sshd_message} %{WORD:user_name}"
           SSHD_INFO_PAM_SESSION_OPENED:
             '%{SSHD_MESSAGE_PAM_SESSION_OPENED:sshd_message} %{WORD:user_name}(\(uid=%{NUMBER:user_id}\))?
@@ -67,7 +76,7 @@ pipeline:
           SSHD_INFO_PAM_IGNORE_MAX_RETRIES: "%{SSHD_MESSAGE_PAM:sshd_message} %{GREEDYDATA:outcome_reason}"
           SSHD_INFO_PAM_WIN_GRANTED_ACCESS: '%{SSHD_MESSAGE_PAM_WIN_GRANTED_ACCESS:sshd_message}: user \''%{USERNAME:user_name}@%{HOSTNAME:source_domain}\'' granted access'
           SSHD_INFO_PAM: "%{SSHD_INFO_PAM_SESSION_CLOSED}|%{SSHD_INFO_PAM_SESSION_OPENED}|%{SSHD_INFO_PAM_MORE_AUTH_FAIL}|%{SSHD_INFO_PAM_IGNORE_MAX_RETRIES}|%{SSHD_INFO_PAM_WIN_GRANTED_ACCESS}"
-          SSHD_INFO: "%{SSHD_INFO_ACCEPTED}|%{SSHD_INFO_DISCONNECTION}|%{SSHD_INFO_PAM}|%{SSHD_EXPIRED_PASSSOWRD}"
+          SSHD_INFO: "%{SSHD_INFO_ACCEPTED}|%{SSHD_INFO_KEY_ACCEPTED}|%{SSHD_INFO_CONNECTION}|%{SSHD_INFO_DISCONNECTION}|%{SSHD_INFO_SESSION_STARTING}|%{SSHD_INFO_USER_ON_PID}|%{SSHD_INFO_RSA_DISABLED}|%{SSHD_INFO_PAM}|%{SSHD_EXPIRED_PASSWORD}"
           SSHD_ERROR_BAD_PROTOCOL:
             "%{SSHD_MESSAGE_BAD_PROTOCOL:sshd_message} identification
             '%{GREEDYDATA}' from %{SSHD_CLIENT_ADDRESS} port %{NUMBER:source_port}"
@@ -104,8 +113,13 @@ pipeline:
           SSHD_ERROR_SEND_FAILURE_PACKET: "%{SSHD_MESSAGE_ERROR_USER:sshd_message} %{GREEDYDATA:outcome_reason}"
           SSHD_ERROR_PAM_CHECK_PASS: "%{SSHD_MESSAGE_PAM_CHECK_PASS:sshd_message} %{SSHD_MESSAGE_USER}"
           SSHD_ERROR_PAM_AUTH_FAILURE: "%{SSHD_MESSAGE_PAM_AUTH_FAILURE:sshd_message} %{GREEDYDATA:pam}"
-          SSHD_ERROR_PAM: "%{SSHD_ERROR_PAM_CHECK_PASS}|%{SSHD_ERROR_PAM_AUTH_FAILURE}"
-          SSHD_EXPIRED_PASSSOWRD:
+          SSHD_ERROR_PAM_FAILED_TO_CREATE: '%{SSHD_MESSAGE_PAMD_FAILED:sshd_message} (?P<outcome_reason>Maximum number of sessions \(%{NUMBER}\) reached, refusing further sessions.)'
+          SSHD_ERROR_PAM_FAULTY_MODULE: "%{SSHD_MESSAGE_PAM_FAULTY_MODULE:sshd_message} %{DATA:filename}"
+          SSHD_ERROR_PAM_UNABLE_DLOPEN: '%{SSHD_MESSAGE_PAM_UNABLE_DLOPEN:sshd_message}\(%{DATA:filename}\): %{DATA:filepath}: %{GREEDYDATA:outcome_reason}'
+          SSHD_ERROR_PAM_MANY_CONSECUTIVE_FAILURES: "%{SSHD_MESSAGE_PAM_FAILLOCK:sshd_message}: (?P<outcome_reason>Consecutive login failures for %{SSHD_MESSAGE_USER} account temporarily locked)"
+          SSHD_ERROR_PAM_USER_UNKNOWN: "%{SSHD_MESSAGE_PAM_FAILLOCK:sshd_message}: (?P<outcome_reason>User unknown)"
+          SSHD_ERROR_PAM: "%{SSHD_ERROR_PAM_CHECK_PASS}|%{SSHD_ERROR_PAM_AUTH_FAILURE}|%{SSHD_ERROR_PAM_FAILED_TO_CREATE}|%{SSHD_ERROR_PAM_FAULTY_MODULE}|%{SSHD_ERROR_PAM_UNABLE_DLOPEN}|%{SSHD_ERROR_PAM_MANY_CONSECUTIVE_FAILURES}|%{SSHD_ERROR_PAM_USER_UNKNOWN}"
+          SSHD_EXPIRED_PASSWORD:
             '%{SSHD_MESSAGE_PAM_EXPIRED_PASSWORD:sshd_message} for user
             %{USERNAME:user_name} \(%{SSHD_TWO_WORDS:action_outcome_reason}\)'
           SSHD_ERROR: "%{SSHD_ERROR_IDENTIFICATION}|%{SSHD_ERROR_INVALID_USER}|%{SSHD_ERROR_NEGOCIATE}|%{SSHD_ERROR_DISCONNECTING_USER}|%{SSHD_ERROR_FOR_USER}|%{SSHD_ERROR_FOR_USER_2}|%{SSHD_ERROR_AUTH_REQUEST}|%{SSHD_ERROR_PAM}|%{SSHD_ERROR_USER_NOT_ALLOWED}|%{SSHD_ERROR_KEX_EXCHANGE_IDENTIFICATION}|%{SSHD_ERROR_CONNECTION}|%{SSHD_ERROR_SEND_FAILURE_PACKET}"
@@ -150,6 +164,11 @@ stages:
           source.ip: "{{parse_kv.message.rhost}}"
         filter: "{{parse_kv.message != None}}"
 
+      - set:
+          destination.ip: "{{event.message.destination_ip}}"
+          destination.port: "{{event.message.destination_port}}"
+        filter: "{{event.message.destination_ip | is_ipaddress }}"
+
       - set:
           event.category: ["authentication"]
           event.type: ["start"]
@@ -159,9 +178,19 @@ stages:
           action.target: "user"
 
           openssh.auth.method: "{{ event.message.outcome_reason }}"
-
         filter: "{{event.message.sshd_message == 'Accepted'}}"
 
+      - set:
+          event.category: ["authentication"]
+          event.type: ["start"]
+          action.type: "open"
+          action.name: "session"
+          action.outcome: "success"
+          action.target: "user"
+
+          openssh.auth.method: "{{ event.message.outcome_reason }}"
+        filter: "{{event.message.sshd_message == 'Postponed'}}"
+
       - set:
           event.category: ["network"]
           event.type: ["end"]
@@ -321,7 +350,6 @@ stages:
           action.name: "sshd:auth"
           action.outcome: "success"
           action.target: "user"
-
         filter: "{{event.message.sshd_message == 'pam_unix(sshd:auth): check pass;'}}"
 
       - set:
@@ -333,7 +361,6 @@ stages:
           action.name: "sshd:auth"
           action.outcome: "failure"
           action.target: "user"
-
         filter: "{{event.message.sshd_message == 'pam_unix(sshd:auth): authentication failure;'}}"
 
       - set:
@@ -373,7 +400,20 @@ stages:
           action.name: "connection"
           action.outcome: "failure"
           action.target: "user"
-        filter: "{{event.message.sshd_message == 'User'}}"
+        filter: "{{event.message.sshd_message == 'User' and 'not allowed' in event.message.outcome_reason}}"
+
+      - set:
+          event.category: ["session"]
+          event.type: ["start"]
+
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.type: "open"
+          action.name: "connection"
+          action.outcome: "success"
+          action.target: "user"
+
+          process.pid: "{{event.message.pid}}"
+        filter: "{{event.message.sshd_message == 'User' and 'is on pid' in event.message.outcome_reason}}"
 
       - set:
           event.category: ["authentication"]
@@ -395,5 +435,73 @@ stages:
           action.name: "connection"
           action.outcome: "failure"
           action.target: "user"
-
         filter: "{{event.message.sshd_message == 'PAM service(sshd)'}}"
+
+      - set:
+          event.category: ["session"]
+          event.type: ["end"]
+
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.type: "open"
+          action.name: "connection"
+          action.outcome: "failure"
+          action.target: "user"
+        filter: "{{event.message.sshd_message == 'pam_systemd(sshd:session): Failed to create session:'}}"
+
+      - set:
+          event.category: ["network"]
+          event.type: ["start"]
+
+          action.type: "open"
+          action.name: "connection"
+          action.outcome: "success"
+          action.target: "user"
+        filter: "{{event.message.sshd_message == 'Connection from'}}"
+
+      - set:
+          event.category: ["library"]
+          event.type: ["start"]
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.outcome: "failure"
+
+          dll.name: "{{event.message.filename}}"
+        filter: "{{event.message.sshd_message == 'PAM adding faulty module:'}}"
+
+      - set:
+          event.category: ["library"]
+          event.type: ["start"]
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.outcome: "failure"
+
+          dll.name: "{{event.message.filename}}"
+          dll.path: "{{event.message.filepath}}"
+        filter: "{{event.message.sshd_message == 'PAM unable to dlopen'}}"
+
+      - set:
+          event.category: ["configuration"]
+          event.type: ["info"]
+
+          action.outcome_reason: "{{ event.message.sshd_message }}"
+        filter: "{{event.message.sshd_message == 'main: sshd: ssh-rsa algorithm is disabled'}}"
+
+      - set:
+          event.category: ["authentication"]
+          event.type: ["end"]
+
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.type: "authentication"
+          action.name: "sshd:auth"
+          action.outcome: "failure"
+          action.target: "user"
+        filter: "{{'Consecutive login failures for' in event.message.outcome_reason}}"
+
+      - set:
+          event.category: ["authentication"]
+          event.type: ["end"]
+
+          action.outcome_reason: "{{ event.message.outcome_reason }}"
+          action.type: "authentication"
+          action.name: "sshd:auth"
+          action.outcome: "failure"
+          action.target: "user"
+        filter: "{{'User unknown' in event.message.outcome_reason}}"

OpenSSH/openssh/tests/accepted_google_authenticator.json

@@ -1,12 +1,6 @@
 {
   "input": {
-    "message": " Accepted google_authenticator for root",
-    "sekoiaio": {
-      "intake": {
-        "dialect": "OpenSSH",
-        "dialect_uuid": "b28db14b-e3a7-463e-8659-9bf0e577944f"
-      }
-    }
+    "message": " Accepted google_authenticator for root"
   },
   "expected": {
     "message": " Accepted google_authenticator for root",

OpenSSH/openssh/tests/accepted_publickey_2.json

@@ -1,12 +1,6 @@
 {
   "input": {
-    "message": " Accepted publickey for [email protected] from 1.2.3.4 port 59294 ssh2: ED25519 SHA256:AbpHGcgLb+kRsJGnwFEktk7uzpZOCcBY74+YBdrKVGs=",
-    "sekoiaio": {
-      "intake": {
-        "dialect": "OpenSSH",
-        "dialect_uuid": "b28db14b-e3a7-463e-8659-9bf0e577944f"
-      }
-    }
+    "message": " Accepted publickey for [email protected] from 1.2.3.4 port 59294 ssh2: ED25519 SHA256:AbpHGcgLb+kRsJGnwFEktk7uzpZOCcBY74+YBdrKVGs="
   },
   "expected": {
     "message": " Accepted publickey for [email protected] from 1.2.3.4 port 59294 ssh2: ED25519 SHA256:AbpHGcgLb+kRsJGnwFEktk7uzpZOCcBY74+YBdrKVGs=",

OpenSSH/openssh/tests/accepted_rsa.json

@@ -0,0 +1,32 @@
+{
+  "input": {
+    "message": "  Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /home/star/.ssh/authorized_keys:2"
+  },
+  "expected": {
+    "message": "  Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /home/star/.ssh/authorized_keys:2",
+    "event": {
+      "category": [
+        "authentication"
+      ],
+      "outcome": "success",
+      "type": [
+        "start"
+      ]
+    },
+    "action": {
+      "name": "session",
+      "outcome": "success",
+      "outcome_reason": "Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /home/star/.ssh/authorized_keys:2",
+      "target": "user",
+      "type": "open"
+    },
+    "observer": {
+      "product": "openssh",
+      "type": "server",
+      "vendor": "openbsd project"
+    },
+    "process": {
+      "name": "sshd"
+    }
+  }
+}

OpenSSH/openssh/tests/accepted_rsa_2.json

@@ -0,0 +1,32 @@
+{
+  "input": {
+    "message": " Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /usr/local/nagios/.ssh/authorized_keys:1"
+  },
+  "expected": {
+    "message": " Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /usr/local/nagios/.ssh/authorized_keys:1",
+    "event": {
+      "category": [
+        "authentication"
+      ],
+      "outcome": "success",
+      "type": [
+        "start"
+      ]
+    },
+    "action": {
+      "name": "session",
+      "outcome": "success",
+      "outcome_reason": "Accepted key RSA SHA256:3cOMdwjvSk5BnU2zs6397YpKn/SNSVSAMtsQchY8dOo found at /usr/local/nagios/.ssh/authorized_keys:1",
+      "target": "user",
+      "type": "open"
+    },
+    "observer": {
+      "product": "openssh",
+      "type": "server",
+      "vendor": "openbsd project"
+    },
+    "process": {
+      "name": "sshd"
+    }
+  }
+}

OpenSSH/openssh/tests/auth_method_disabled.json

@@ -0,0 +1,28 @@
+{
+  "input": {
+    "message": "main: sshd: ssh-rsa algorithm is disabled"
+  },
+  "expected": {
+    "message": "main: sshd: ssh-rsa algorithm is disabled",
+    "event": {
+      "category": [
+        "configuration"
+      ],
+      "outcome": "success",
+      "type": [
+        "info"
+      ]
+    },
+    "action": {
+      "outcome_reason": "main: sshd: ssh-rsa algorithm is disabled"
+    },
+    "observer": {
+      "product": "openssh",
+      "type": "server",
+      "vendor": "openbsd project"
+    },
+    "process": {
+      "name": "sshd"
+    }
+  }
+}

OpenSSH/openssh/tests/authentication_attempts_exceeded_2.json

@@ -1,12 +1,6 @@
 {
   "input": {
-    "message": " error: maximum authentication attempts exceeded for root from 1.2.3.4 port 63758 ssh2 [preauth]",
-    "sekoiaio": {
-      "intake": {
-        "dialect": "OpenSSH",
-        "dialect_uuid": "b28db14b-e3a7-463e-8659-9bf0e577944f"
-      }
-    }
+    "message": " error: maximum authentication attempts exceeded for root from 1.2.3.4 port 63758 ssh2 [preauth]"
   },
   "expected": {
     "message": " error: maximum authentication attempts exceeded for root from 1.2.3.4 port 63758 ssh2 [preauth]",