Merge branch 'main' into bfilar.nlu_signals

.github/workflows/rule-validate.yml

@@ -30,6 +30,7 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
+          depth: 0
 
       - uses: actions/setup-python@v4
         with:
@@ -92,6 +93,45 @@ jobs:
         id: get_head
         run: echo "##[set-output name=HEAD;]$(git rev-parse HEAD)"
 
+      - name: Get changed detection-rules
+        id: changed-files
+        uses: tj-actions/changed-files@v39
+        with:
+          files: "detection-rules/**"
+          recover_deleted_files: true
+
+      - name: "Find updated rule IDs"
+        id: find_ids
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do
+            echo "$file was changed"
+            rule_id=$(yq '.id' $file)
+          
+            echo "$file has rule ID $rule_id"
+            altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
+          done
+          
+          echo "Altered Ruled IDs: [$altered_rule_ids]"
+          echo "##[set-output name=rule_ids;]$(echo $altered_rule_ids)"
+          # TODO: This doesn't solve for a modified rule_id. We could merge with any files known on 'main', but changing
+          # a rule ID is a separate problem.
+
+      - name: "Trigger MQL Mimic Tests"
+        env:
+          trigger_url: '${{ secrets.MQL_MOCK_TRIGGER }}'
+          branch: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref }}
+          repo: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name || github.repository }}
+          token: '${{ secrets.GITHUB_TOKEN }}'
+          sha: '${{ steps.get_head.outputs.HEAD }}'
+          only_rule_ids: '${{ steps.find_ids.outputs.rule_ids }}'
+        run: |
+          body='{"branch":"'$branch'","repo":"'$repo'","token":"'$token'","sha":"'$sha'","only_rule_ids":"'$only_rule_ids'"}'
+          echo $body
+          
+          curl -X POST $trigger_url  \
+            -H 'Content-Type: application/json' \
+            -d "$body"
+
       # When we add a commit, GitHub won't trigger actions on the auto commit, so we're missing a required check on the
       # HEAD commit.
       # Various alternatives were explored, but all run into issues when dealing with forks. This sets a "Check" for
@@ -126,3 +166,14 @@ jobs:
                 text: "Rule Tests and ID Updated",
               },
             });
+
+      - name: Wait for MQL Mimic check to be completed
+        uses: fountainhead/[email protected]
+        id: wait-for-build
+        # Wait for results so that the token remains valid while the test suite is executing and posting a check here.
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          checkName: "MQL Mimic Tests"
+          ref: ${{ steps.get_head.outputs.HEAD }}
+          timeoutSeconds: 3600
+

.github/workflows/update-test-rules.yml

@@ -104,12 +104,13 @@ jobs:
           done
           
           # This workflow is trigerred from an issue comment so we don't automatically have context on what changed in the PR
+          # TODO: We can only retrieve 100 results, we need to add pagination support.
           curl -L \
             -o pr_files.json \
             -H "Accept: application/vnd.github+json" \
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/sublime-security/sublime-rules/pulls/$pr_num/files
+            https://api.github.com/repos/sublime-security/sublime-rules/pulls/$pr_num/files?per_page=100
           
           # Any files added/changed/modified will be copied to test-rules
           files_changed=$(jq -r '.[] | select(.status == "added" or .status == "modified" or .status == "changed") | .filename' pr_files.json)
@@ -122,6 +123,13 @@ jobs:
           # but it doesn't seem any simpler). And then add testing metadata.
           # If multiple PRs modify the same file, only one can be tested. This is solveable, but not something we see often.
           for file in $files_changed; do
+            # Skip any LA rules. We'll ignore these downstream anyway, but best to keep the branch clean.
+            la_count=$(grep -c 'beta.linkanalysis' source/$file || true)
+            if [[ "$la_count" != '0' ]]; then
+              echo "Ignoring $file because of linkanalysis usage"
+              continue
+            fi
+            
             cp source/$file destination/$file
             yq -i '.testing_pr = env(pr_num)' destination/$file
             yq -i '.testing_sha = env(sha)' destination/$file

detection-rules/attachment_adobe_image_lure_fts.yml

@@ -20,13 +20,10 @@ source: |
           )
   )
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $sender_emails
-    )
+    profile.by_sender().prevalence in ("new", "outlier")
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $sender_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 attack_types:

detection-rules/attachment_any_html_in_archive_unsolicited.yml

@@ -14,13 +14,10 @@ source: |
           and any(file.explode(.), .depth > 0 and .file_extension in~ ("html", "htm"))
   )
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 tags:

detection-rules/attachment_any_html_new_sender.yml

@@ -11,16 +11,11 @@ severity: "medium"
 source: |
   type.inbound
   and any(attachments, .file_extension in~ ('htm', 'html') or .file_type == "html")
-
-  // first-time sender
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $sender_emails
-    )
+    profile.by_sender().prevalence in ("new", "outlier")
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $sender_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 tags:

detection-rules/attachment_any_html_unsolicited.yml

@@ -11,16 +11,11 @@ severity: "low"
 source: |
   type.inbound
   and any(attachments, .file_extension in~ ('htm', 'html') or .file_type == "html")
-
-  // unsolicited
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 tags:

detection-rules/attachment_callback_phish_with_img.yml

@@ -10,13 +10,10 @@ severity: "high"
 source: |
   type.inbound
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
   and sender.email.domain.root_domain in $free_email_providers

detection-rules/attachment_callback_phish_with_pdf.yml

@@ -8,13 +8,10 @@ severity: "high"
 source: |
   type.inbound
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 

detection-rules/attachment_cve_2023_38831.yml

@@ -0,0 +1,67 @@
+name: "Attachment: Zip Exploiting CVE-2023-38831 (Unsolicited)"
+description: |
+  A Zip attachment that exhibits attributes required to exploit CVE-2023-38831, a vulnerability in WinRAR (prior to 6.23).
+type: "rule"
+severity: "critical"
+authors:
+  - twitter: "delivr_to"
+references:
+  - https://twitter.com/GroupIB_TI/status/1694277126944633328
+  - https://www.group-ib.com/blog/cve-2023-38831-winrar-zero-day/
+  - https://github.com/b1tg/CVE-2023-38831-winrar-exploit/
+  - https://delivr.to/payloads?id=ab969e8a-bf5c-45a6-acd0-0dd2b2a34750
+source: |
+  type.inbound
+  and any(attachments,
+      .file_extension in $file_extensions_common_archives and
+      any(file.explode(.), 
+          (
+              .depth == 0 and 
+              any(.scan.zip.all_paths, 
+                  regex.match(., 
+                  // zip contains a path with spaces and file extensions 
+                  // lure.pdf /lure.pdf .cmd
+                  // 
+                  //  /= Initial file name
+                  //  |
+                  //  |        /= Space
+                  //  |        |
+                  //  |        | /= Folder
+                  //  |        | |
+                  //  |        | | /= Repeated file name
+                  //  |        | | |
+                  //  |        | | |       /= Space
+                  //  |        | | |       |   
+                  //  |        | | |       |   /= Real script ending
+                  //  |        | | |       |   |
+                      '\w+\.\w+\s\/\w+\.\w+\s\.\w+'
+                  )
+              )
+          ) and 
+          (
+              // One file name is present in another, e.g.
+              //     delivrto.pdf 
+              //     delivrto.pdf /delivrto.pdf .cmd
+              any(.scan.zip.all_paths, 
+                  any(..scan.zip.all_paths,
+                      . != .. and 
+                      strings.starts_with(., ..)
+                  )
+              )
+          )
+      )
+  )
+  and (
+      (
+          sender.email.domain.root_domain in $free_email_providers
+          and sender.email.email not in $recipient_emails
+      )
+      or (
+          sender.email.domain.root_domain not in $free_email_providers
+          and sender.email.domain.domain not in $recipient_domains
+      )
+  )
+tags:
+  - "Suspicious Attachment"
+  - "CVE-2023-38831"
+id: "926b96ae-f40b-525d-a312-bd6c9a5f19fb"

detection-rules/attachment_docusign_image_suspicious_links.yml

@@ -26,13 +26,11 @@ source: |
           )
   )
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $sender_emails
-    )
+    profile.by_sender().prevalence in ("new", "outlier")
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $sender_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
+
     )
   )
 attack_types:

detection-rules/attachment_dropbox_image_suspicious_links.yml

@@ -14,13 +14,10 @@ source: |
           )
   )
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $sender_emails
-    )
+    profile.by_sender().prevalence in ("new", "outlier")
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $sender_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 attack_types:

detection-rules/attachment_eml_cred_theft.yml

@@ -29,16 +29,11 @@ source: |
   and not any(attachments, .content_type == "message/delivery-status")
   // if the "References" is in the body of the message, it's probably a bounce
   and not any(headers.references, strings.contains(body.html.display_text, .))
-
-  // unsolicited
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 attack_types:

detection-rules/attachment_eml_with_html_attachment.yml

@@ -40,16 +40,11 @@ source: |
   and not any(attachments, .content_type == "message/delivery-status")
   // if the "References" is in the body of the message, it's probably a bounce
   and not any(headers.references, strings.contains(body.html.display_text, .))
-
-  // unsolicited
   and (
-    (
-      sender.email.domain.root_domain in $free_email_providers
-      and sender.email.email not in $recipient_emails
-    )
+    not profile.by_sender().solicited
     or (
-      sender.email.domain.root_domain not in $free_email_providers
-      and sender.email.domain.domain not in $recipient_domains
+      profile.by_sender().any_messages_malicious_or_spam
+      and not profile.by_sender().any_false_positives
     )
   )
 tags: