diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 501462fb5..9609d154e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -17,13 +17,13 @@ jobs: steps: - name: Checkout main - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive path: main - name: Checkout hayabusa-sample-evtx repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: Yamato-Security/hayabusa-sample-evtx path: hayabusa-sample-evtx diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 9fc5bf772..95e93a022 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -1,5 +1,15 @@ # 変更点 +## x.x.x [xxxx/xx/xx] + +**新機能:** + +- デフォルトでは、`.evtx`ファイルに適用可能なルールのみ有効になる。これは、`.evtx`ファイルと`.yml`ルールの`Channel`フィールドに基づく。例えば、`Security.evtx`がスキャンされている場合、`Channel: Security`が定義されているルールのみがこのファイルに対して使用される。ベンチマークでは、単一の`evtx`ファイルをスキャンする場合、パフォーマンスが約20％向上される。1つの`.evtx`ファイルで複数のチャネルが使用されている場合や、チャネルが定義されていないルールを使用して、チャネルに関係なくすべての`.evtx`ファイルをスキャンしたい場合は、`csv-timeline` と `json-timeline` の `-A、--enable-all-rules` オプションでこのフィルタリングをオフにすることができる。（#1317）(@fukusuket) + - 現在のところ、`Channel`が定義されておらず、すべての`.evtx`ファイルをスキャンすることを意図している検知ルールは以下の2つだけ: + - [Possible Hidden Shellcode](https://github.com/Yamato-Security/hayabusa-rules/blob/main/hayabusa/builtin/UnkwnChannEID_Med_PossibleHiddenShellcode.yml) + - [Mimikatz Use](https://github.com/SigmaHQ/sigma/blob/master/rules/windows/builtin/win_alert_mimikatz_keywords.yml) +- デフォルトでは、適用可能なルールを持つ`.evtx`ファイルのみ読み込む。たとえば、さまざまなイベントログのディレクトリをスキャンしている場合でも、 `Channel: Security` を探すルールのみを有効にした場合、Hayabusaは`Security`以外のすべてのイベントログを無視します。ベンチマークでは、通常のスキャンで約10％、単一のルールでスキャンする場合は最大60％以上のパフォーマンス向上が得られる。チャネルに関係なくすべての`.evtx`ファイルを読み込みたい場合は、`csv-timeline` と `json-timeline` の `-a、--scan-all-evtx-files` オプションでこのフィルタリングをオフにすることができる。(#1318) (@fukusuket) + ## 2.15.0 [2024/04/20] "Sonic Release" **改善:** @@ -723,4 +733,4 @@ ## v1.0.0 [2021/12/25] -- 最初のリリース +- 最初のリリース \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ede0dc8..3d26c6466 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changes +## x.x.x [xxxx/xx/xx] + +**New Features:** + +- By default now, only rules that are applicable to loaded evtx files will be enabled. This is based on the `Channel` field in `.evtx` file and `.yml` rule. For example, if `Security.evtx` was being scanned, then only rules that have `Channel: Security` defined will be used against this file. In our benchmarks, this gives a speed benefit of around 20% when scanning single `evtx` files. If you think there are multiple channels being used in a single `.evtx` file or you want to use rules that do not have the `Channel` field defined in order to scan all `.evtx` files regardless of the channel, then you can turn off this filtering with the `-A, --enable-all-rules` option in `csv-timeline` and `json-timeline`. (#1317) (@fukusuket) + - Currently, the only two detection rules that do not have `Channel` defined and are intended to scan all `.evtx` files are the following: + - [Possible Hidden Shellcode](https://github.com/Yamato-Security/hayabusa-rules/blob/main/hayabusa/builtin/UnkwnChannEID_Med_PossibleHiddenShellcode.yml) + - [Mimikatz Use](https://github.com/SigmaHQ/sigma/blob/master/rules/windows/builtin/win_alert_mimikatz_keywords.yml) +- By default now, `.evtx` files that have applicable rules will be loaded. So for example, if you are scanning a directory of various event logs but only enable a rule that is looking for `Channel: Security` then Hayabusa will ignore all non-security event logs. In our benchmarks, this gives a speed benefit of around 10% with normal scans and up to 60%+ performance increase when scanning with a single rule. If you want to load all `.evtx` files regardless of channel, then you can turn off this filtering with the `-a, --scan-all-evtx-files` option in `csv-timeline` and `json-timeline`. (#1318) (@fukusuket) + ## 2.15.0 [2024/04/20] "Sonic Release" **Enhancements:** diff --git a/Cargo.lock b/Cargo.lock index c3f095541..e7998906e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,15 +153,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -247,12 +241,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.94" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -493,7 +488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -638,7 +633,7 @@ dependencies = [ "crc32fast", "dialoguer", "encoding", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "indoc", "jemallocator", "log", @@ -654,9 +649,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "file-chunker" @@ -670,9 +665,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -748,9 +743,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -758,10 +753,10 @@ dependencies = [ [[package]] name = "hayabusa" -version = "2.15.0" +version = "2.16.0-dev" dependencies = [ "aho-corasick", - "base64 0.22.0", + "base64", "bytesize", "chrono", "cidr-utils", @@ -775,7 +770,7 @@ dependencies = [ "downcast-rs", "evtx", "git2", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "hex", "horrorshow", "indexmap 2.2.6", @@ -895,7 +890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -981,9 +976,9 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -1023,9 +1018,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libgit2-sys" @@ -1043,9 +1038,9 @@ dependencies = [ [[package]] name = "libmimalloc-sys" -version = "0.1.35" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" +checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" dependencies = [ "cc", "cty", @@ -1092,9 +1087,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1144,9 +1139,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.39" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c" +checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" dependencies = [ "libmimalloc-sys", ] @@ -1366,9 +1361,9 @@ checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", "parking_lot_core", @@ -1376,15 +1371,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -1507,11 +1502,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] @@ -1586,9 +1581,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -1613,15 +1608,15 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ "ring", "rustls-pki-types", @@ -1666,18 +1661,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", "quote", @@ -1703,9 +1698,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -1733,9 +1728,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1851,18 +1846,18 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", @@ -1946,9 +1941,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "untrusted" @@ -1958,11 +1953,11 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.6" +version = "2.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35" +checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" dependencies = [ - "base64 0.21.7", + "base64", "flate2", "log", "once_cell", @@ -2105,11 +2100,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dcaf76b96..e21f2a907 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hayabusa" -version = "2.15.0" +version = "2.16.0-dev" repository = "https://github.com/Yamato-Security/hayabusa" authors = ["Yamato Security @SecurityYamato"] edition = "2021" diff --git a/src/afterfact.rs b/src/afterfact.rs index e20719b86..1cdd3d463 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -2295,6 +2295,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -2386,6 +2388,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2631,6 +2635,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_multiline.csv").to_path_buf()), @@ -2732,6 +2738,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2961,6 +2969,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_remove_duplicate.csv").to_path_buf()), @@ -3052,6 +3062,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -3292,6 +3304,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_remove_duplicate.json").to_path_buf()), @@ -3383,6 +3397,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -3696,6 +3712,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_multiple_data_in_details.json").to_path_buf()), @@ -3788,6 +3806,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -4046,6 +4066,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_json.json").to_path_buf()), @@ -4138,6 +4160,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -4321,6 +4345,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_jsonl.jsonl").to_path_buf()), @@ -4413,6 +4439,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }; let ch = mock_ch_filter .get(&CompactString::from("security")) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 84a8a4d2e..cd2d77220 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -91,6 +91,8 @@ pub struct StoredStatic { pub enable_recover_records: bool, pub timeline_offset: Option, pub is_low_memory: bool, + pub enable_all_rules: bool, + pub scan_all_evtx_files: bool, } impl StoredStatic { /// main.rsでパースした情報からデータを格納する関数 @@ -562,6 +564,16 @@ impl StoredStatic { Some(Action::JsonTimeline(opt)) => opt.output_options.low_memory_mode, _ => false, }; + let enable_all_rules = match &input_config.as_ref().unwrap().action { + Some(Action::CsvTimeline(opt)) => opt.output_options.enable_all_rules, + Some(Action::JsonTimeline(opt)) => opt.output_options.enable_all_rules, + _ => false, + }; + let scan_all_evtx_files = match &input_config.as_ref().unwrap().action { + Some(Action::CsvTimeline(opt)) => opt.output_options.scan_all_evtx_files, + Some(Action::JsonTimeline(opt)) => opt.output_options.scan_all_evtx_files, + _ => false, + }; let mut ret = StoredStatic { config: input_config.as_ref().unwrap().to_owned(), config_path: config_path.to_path_buf(), @@ -678,6 +690,8 @@ impl StoredStatic { timeline_offset, include_status, is_low_memory, + enable_all_rules, + scan_all_evtx_files, }; ret.profiles = load_profile( check_setting_path( @@ -774,7 +788,7 @@ fn check_thread_number(config: &Config) -> Option { pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe csv-timeline [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe csv-timeline [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 290, disable_help_flag = true @@ -784,7 +798,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe json-timeline [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe json-timeline [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 360, disable_help_flag = true @@ -794,7 +808,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe logon-summary [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe logon-summary [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 383, disable_help_flag = true @@ -804,7 +818,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe eid-metrics [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe eid-metrics [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 310, disable_help_flag = true @@ -814,7 +828,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe pivot-keywords-list [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe pivot-keywords-list [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 420, disable_help_flag = true @@ -824,7 +838,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe search <--keywords \"\" OR --regex \"\"> [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe search <--keywords \"\" OR --regex \"\"> [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 450, disable_help_flag = true @@ -834,7 +848,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", term_width = 400, display_order = 470, disable_help_flag = true @@ -844,7 +858,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", term_width = 400, display_order = 380, disable_help_flag = true @@ -854,7 +868,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n {usage}\n\n{all-args}", term_width = 400, display_order = 451, disable_help_flag = true @@ -872,7 +886,7 @@ pub enum Action { #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe computer-metrics [OPTIONS]\n\n{all-args}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe computer-metrics [OPTIONS]\n\n{all-args}", term_width = 400, display_order = 290, disable_help_flag = true @@ -1109,8 +1123,8 @@ pub struct SearchOption { #[arg(help_heading = Some("Time Format"), long = "European-time", display_order = 50)] pub european_time: bool, - /// Output timestamp in ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) - #[arg(help_heading = Some("Time Format"), long = "ISO-8601", display_order = 90)] + /// Output timestamp in original ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) + #[arg(help_heading = Some("Time Format"), short = 'O', long = "ISO-8601", display_order = 90)] pub iso_8601: bool, /// Output timestamp in RFC 2822 format (ex: Fri, 22 Feb 2022 22:00:00 -0600) @@ -1190,8 +1204,8 @@ pub struct EidMetricsOption { #[arg(help_heading = Some("Time Format"), long = "European-time", display_order = 50)] pub european_time: bool, - /// Output timestamp in ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) - #[arg(help_heading = Some("Time Format"), long = "ISO-8601", display_order = 90)] + /// Output timestamp in original ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) + #[arg(help_heading = Some("Time Format"), short = 'O', long = "ISO-8601", display_order = 90)] pub iso_8601: bool, /// Output timestamp in RFC 2822 format (ex: Fri, 22 Feb 2022 22:00:00 -0600) @@ -1338,8 +1352,8 @@ pub struct LogonSummaryOption { #[arg(help_heading = Some("Time Format"), long = "European-time", display_order = 50)] pub european_time: bool, - /// Output timestamp in ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) - #[arg(help_heading = Some("Time Format"), long = "ISO-8601", display_order = 90)] + /// Output timestamp in original ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) + #[arg(help_heading = Some("Time Format"), short = 'O', long = "ISO-8601", display_order = 90)] pub iso_8601: bool, /// Output timestamp in RFC 2822 format (ex: Fri, 22 Feb 2022 22:00:00 -0600) @@ -1481,8 +1495,8 @@ pub struct OutputOption { #[arg(help_heading = Some("Time Format"), long = "European-time", display_order = 50)] pub european_time: bool, - /// Output timestamp in ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) - #[arg(help_heading = Some("Time Format"), long = "ISO-8601", display_order = 90)] + /// Output timestamp in original ISO-8601 format (ex: 2022-02-22T10:10:10.1234567Z) (Always UTC) + #[arg(help_heading = Some("Time Format"), short = 'O', long = "ISO-8601", display_order = 90)] pub iso_8601: bool, /// Output timestamp in RFC 2822 format (ex: Fri, 22 Feb 2022 22:00:00 -0600) @@ -1563,6 +1577,14 @@ pub struct OutputOption { /// Scan with the minimal amount of memory by not sorting events #[arg(help_heading = Some("General Options"), short='s', long = "low-memory-mode", display_order = 380)] pub low_memory_mode: bool, + + /// Enable all rules regardless of loaded evtx files + #[arg(help_heading = Some("Filtering"), short='A', long = "enable-all-rules", display_order = 300)] + pub enable_all_rules: bool, + + /// Scan all evtx files regardless of loaded rules + #[arg(help_heading = Some("Filtering"), short='a', long = "scan-all-evtx-files", display_order = 450)] + pub scan_all_evtx_files: bool, } #[derive(Copy, Args, Clone, Debug)] @@ -1711,7 +1733,7 @@ pub struct ComputerMetricsOption { #[derive(Parser, Clone, Debug)] #[clap( author = "Yamato Security (https://github.com/Yamato-Security/hayabusa - @SecurityYamato)", - help_template = "\nHayabusa v2.15.0 - Sonic Release\n{author-with-newline}\n{usage-heading}\n hayabusa.exe [OPTIONS]\n hayabusa.exe help \n\n{all-args}{options}", + help_template = "\nHayabusa v2.16.0 - Dev Build\n{author-with-newline}\n{usage-heading}\n hayabusa.exe [OPTIONS]\n hayabusa.exe help or hayabusa.exe -h\n\n{all-args}{options}", term_width = 400, disable_help_flag = true )] @@ -2248,6 +2270,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: option.no_wizard, include_status: option.include_status.clone(), low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::EidMetrics(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2289,6 +2313,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::LogonSummary(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2330,6 +2356,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::ComputerMetrics(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2380,6 +2408,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::Search(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -2430,6 +2460,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::SetDefaultProfile(option) => Some(OutputOption { input_args: InputOption { @@ -2486,6 +2518,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), Action::UpdateRules(option) => Some(OutputOption { input_args: InputOption { @@ -2542,6 +2576,8 @@ fn extract_output_options(config: &Config) -> Option { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }), _ => None, } @@ -2795,6 +2831,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -2871,6 +2909,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 355aa5a2f..dadd8c838 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1,9 +1,7 @@ extern crate csv; use crate::detections::configs::Action; -use crate::detections::utils::{ - create_recordinfos, format_time, output_profile_name, write_color_buffer, -}; +use crate::detections::utils::{create_recordinfos, format_time, write_color_buffer}; use crate::options::profile::Profile::{ self, Channel, Computer, EventID, EvtxFile, Level, MitreTactics, MitreTags, OtherTags, Provider, RecordID, RecoveredRecord, RenderedMessage, RuleAuthor, RuleCreationDate, RuleFile, @@ -1164,15 +1162,11 @@ impl Detection { }); let tmp_total_detect_output = format!( - "Total enabled detection rules: {}", + "Total detection rules: {}", total_loaded_rule_cnt.to_formatted_string(&Locale::en) ); println!("{tmp_total_detect_output}"); println!(); - output_profile_name(&stored_static.output_option, true); - println!(); - println!("Scanning in progress. Please wait."); - println!(); if stored_static.html_report_flag { html_report_stock.push(format!("- {tmp_total_detect_output}")); } @@ -1296,6 +1290,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -1558,6 +1554,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1695,6 +1693,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1827,6 +1827,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1975,6 +1977,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), diff --git a/src/detections/rule/condition_parser.rs b/src/detections/rule/condition_parser.rs index e018cc725..b9ba3d9c7 100644 --- a/src/detections/rule/condition_parser.rs +++ b/src/detections/rule/condition_parser.rs @@ -509,6 +509,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/count.rs b/src/detections/rule/count.rs index 82f365ef3..b6c68abae 100644 --- a/src/detections/rule/count.rs +++ b/src/detections/rule/count.rs @@ -637,6 +637,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/matchers.rs b/src/detections/rule/matchers.rs index ceb4817ad..ceb3e0ce6 100644 --- a/src/detections/rule/matchers.rs +++ b/src/detections/rule/matchers.rs @@ -921,6 +921,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/mod.rs b/src/detections/rule/mod.rs index a1e12ba43..8b9fcbc72 100644 --- a/src/detections/rule/mod.rs +++ b/src/detections/rule/mod.rs @@ -450,6 +450,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/rule/selectionnodes.rs b/src/detections/rule/selectionnodes.rs index 2eff7c07d..2e10e2496 100644 --- a/src/detections/rule/selectionnodes.rs +++ b/src/detections/rule/selectionnodes.rs @@ -577,6 +577,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/detections/utils.rs b/src/detections/utils.rs index 4d6a18869..5b44a8b1c 100644 --- a/src/detections/utils.rs +++ b/src/detections/utils.rs @@ -1076,6 +1076,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/filter.rs b/src/filter.rs index 2f37d1397..0d84ef866 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -1,9 +1,14 @@ use crate::detections::configs::{self, StoredStatic}; use crate::detections::message::{AlertMessage, ERROR_LOG_STACK}; +use crate::detections::rule::RuleNode; +use evtx::EvtxParser; use hashbrown::HashMap; use regex::Regex; +use std::collections::HashSet; use std::fs::File; use std::io::{BufRead, BufReader}; +use std::path::PathBuf; +use yaml_rust::Yaml; #[derive(Debug)] pub struct DataFilterRule { @@ -79,3 +84,258 @@ impl RuleExclude { } } } + +fn peek_channel_from_evtx_first_record( + evtx_files: &Vec, +) -> Result>, Box> { + let mut channels = HashMap::new(); + for path in evtx_files { + let mut parser = EvtxParser::from_path(path)?; + let mut records = parser.records_json_value(); + match records.next() { + Some(Ok(rec)) => { + let key = rec.data["Event"]["System"]["Channel"] + .as_str() + .unwrap_or("") + .trim_matches('"') + .to_string(); + channels + .entry(key) + .or_insert_with(Vec::new) + .push(path.to_path_buf()); + } + _ => continue, + }; + } + Ok(channels) +} + +fn extract_channel_from_rules( + rule_files: &Vec, + evtx_channels: &HashSet, +) -> (Vec, Vec) { + fn visit_value( + key: &str, + value: &Yaml, + evtx_channels: &HashSet, + intersection_channels: &mut Vec, + ) { + match *value { + Yaml::String(ref s) => { + if key == "Channel" { + if s.contains('*') { + // SigmaルールでChannelにワイルドカードが使われた場合 + for ch in evtx_channels { + if ch.contains(s.trim_matches('*')) { + intersection_channels.push(ch.to_string()); + } + } + } else if evtx_channels.contains(s) { + intersection_channels.push(s.clone()); + } + } + } + Yaml::Hash(ref map) => { + for (k, v) in map { + visit_value(k.as_str().unwrap(), v, evtx_channels, intersection_channels); + } + } + Yaml::Array(ref seq) => { + for v in seq { + visit_value(key, v, evtx_channels, intersection_channels); + } + } + _ => {} + } + } + let mut intersection_channels = vec![]; + let mut filtered_rulespathes = vec![]; + for rule in rule_files { + let before_visit_len = intersection_channels.len(); + visit_value("", &rule.yaml, evtx_channels, &mut intersection_channels); + if before_visit_len < intersection_channels.len() { + filtered_rulespathes.push(rule.rulepath.to_string()); + } + } + (filtered_rulespathes, intersection_channels) +} + +pub struct ChannelFilter { + pub rulepathes: Vec, + pub intersec_channels: HashSet, // evtxとruleのchannelの積集合 + pub evtx_channels_map: HashMap>, // key=channel, val=evtxパスのリスト +} + +impl ChannelFilter { + pub fn new() -> ChannelFilter { + ChannelFilter { + rulepathes: vec![], + intersec_channels: HashSet::new(), + evtx_channels_map: HashMap::new(), + } + } + + pub fn scanable_rule_exists(&mut self, path: &PathBuf) -> bool { + for (channel, rulepathes) in &self.evtx_channels_map { + if rulepathes.contains(path) && self.intersec_channels.contains(channel) { + return true; + } + } + false + } +} + +impl Default for ChannelFilter { + fn default() -> Self { + Self::new() + } +} + +pub fn create_channel_filter( + evtx_files: &Vec, + rule_nodes: &Vec, +) -> ChannelFilter { + let channels = peek_channel_from_evtx_first_record(evtx_files); + match channels { + Ok(ch) => { + let (x, y) = extract_channel_from_rules(rule_nodes, &ch.keys().cloned().collect()); + ChannelFilter { + rulepathes: x, + intersec_channels: y.into_iter().collect(), + evtx_channels_map: ch, + } + } + _ => ChannelFilter::new(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use yaml_rust::YamlLoader; + + #[test] + fn test_channel_filter_scanable_rule_exists() { + let mut channel_filter = ChannelFilter::new(); + channel_filter + .evtx_channels_map + .insert("channel1".to_string(), vec![PathBuf::from("path1")]); + channel_filter + .intersec_channels + .insert("channel1".to_string()); + + assert!(channel_filter.scanable_rule_exists(&PathBuf::from("path1"))); + assert!(!channel_filter.scanable_rule_exists(&PathBuf::from("path2"))); + } + + #[test] + fn test_peek_channel_from_evtx_first_record_invalid_evtx() { + let evtx_files = vec![PathBuf::from("test_files/evtx/test1.evtx")]; + let result = peek_channel_from_evtx_first_record(&evtx_files); + assert!(result.is_err()); + } + + #[test] + fn test_extract_channel_from_rules_hash_match() { + let rule_str = r#" + detection: + selection1: + Channel: 'Microsoft-Windows-Sysmon/Operational' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_files = vec![rule]; + let evtx_channels = HashSet::from_iter(vec!["Microsoft-Windows-Sysmon/Operational".into()]); + let (result, _) = extract_channel_from_rules(&rule_files, &evtx_channels); + assert_eq!(result, vec!["test_files/evtx/test1.evtx"]); + } + + #[test] + fn test_extract_channel_from_rules_hash_wildcard_match() { + let rule_str = r#" + detection: + selection1: + Channel: 'Microsoft-Windows-Security-Mitigations*' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_files = vec![rule]; + let evtx_channels = HashSet::from_iter(vec![ + "Microsoft-Windows-Security-Mitigations%4KernelMode.evtx".into(), + "Microsoft-Windows-Security-Mitigations%4UserMode.evtx".into(), + ]); + let (result, _) = extract_channel_from_rules(&rule_files, &evtx_channels); + assert_eq!(result, vec!["test_files/evtx/test1.evtx"]); + } + + #[test] + fn test_extract_channel_from_rules_hash_not_match() { + let rule_str = r#" + detection: + selection1: + Channel: 'Security' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_files = vec![rule]; + let evtx_channels = HashSet::from_iter(vec!["Microsoft-Windows-Sysmon/Operational".into()]); + let (result, _) = extract_channel_from_rules(&rule_files, &evtx_channels); + assert_eq!(result.len(), 0); + } + + #[test] + fn test_extract_channel_from_rules_array_match() { + let rule_str = r#" + detection: + selection1: + Channel: + - 'Security' + - 'Microsoft-Windows-Sysmon/Operational' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_files = vec![rule]; + let evtx_channels = HashSet::from_iter(vec!["Microsoft-Windows-Sysmon/Operational".into()]); + let (result, _) = extract_channel_from_rules(&rule_files, &evtx_channels); + assert_eq!(result, vec!["test_files/evtx/test1.evtx"]); + } + + #[test] + fn test_extract_channel_from_rules_array_not_match() { + let rule_str = r#" + detection: + selection1: + Channel: + - 'Security' + - 'System' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_files = vec![rule]; + let evtx_channels = HashSet::from_iter(vec!["Microsoft-Windows-Sysmon/Operational".into()]); + let (result, _) = extract_channel_from_rules(&rule_files, &evtx_channels); + assert_eq!(result.len(), 0); + } + + #[test] + fn test_filter_rules_by_evtx_channel_invalid_evtx() { + let evtx_files = vec![PathBuf::from("test_files/evtx/test1.evtx")]; + let rule_str = r#" + detection: + selection1: + Channel: 'Microsoft-Windows-Sysmon/Operational' + "#; + let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter(); + let test_yaml_data = rule_yaml.next().unwrap(); + let rule = RuleNode::new("test_files/evtx/test1.evtx".to_string(), test_yaml_data); + let rule_nodes = vec![rule]; + let result = create_channel_filter(&evtx_files, &rule_nodes); + assert_eq!(result.rulepathes.len(), 0); + } +} diff --git a/src/main.rs b/src/main.rs index 018b82172..01ebd9ede 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ use hayabusa::detections::utils; use hayabusa::detections::utils::{ check_setting_path, get_writable_color, output_and_data_stack_for_html, output_profile_name, }; +use hayabusa::filter::create_channel_filter; use hayabusa::options::htmlreport::{self, HTML_REPORTER}; use hayabusa::options::pivot::create_output; use hayabusa::options::pivot::PIVOT_KEYWORD; @@ -977,7 +978,7 @@ impl App { fn analysis_files( &mut self, - evtx_files: Vec, + mut evtx_files: Vec, time_filter: &TargetEventTime, stored_static: &mut StoredStatic, ) { @@ -1377,7 +1378,6 @@ impl App { .unwrap() .min_level .to_uppercase(); - println!(); if !(stored_static.logon_summary_flag || stored_static.search_flag @@ -1430,6 +1430,32 @@ impl App { .ok(); return; } + if !stored_static.scan_all_evtx_files && !stored_static.enable_all_rules { + println!("Creating the channel filter. Please wait."); + println!(); + let mut channel_filter = create_channel_filter(&evtx_files, &rule_files); + if !stored_static.scan_all_evtx_files { + evtx_files.retain(|e| channel_filter.scanable_rule_exists(e)); + let evtx_files_after_channel_filter = format!( + "Evtx files loaded after channel filter: {}", + (evtx_files.len()).to_formatted_string(&Locale::en) + ); + println!("{evtx_files_after_channel_filter}"); + } + if !stored_static.enable_all_rules { + rule_files.retain(|r| channel_filter.rulepathes.contains(&r.rulepath)); + let rules_after_channel_filter = format!( + "Detection rules enabled after channel filter: {}", + (rule_files.len()).to_formatted_string(&Locale::en) + ); + println!("{rules_after_channel_filter}"); + println!(); + } + } + output_profile_name(&stored_static.output_option, true); + println!(); + println!("Scanning in progress. Please wait."); + println!(); } let template = if stored_static.common_options.no_color { @@ -1452,6 +1478,7 @@ impl App { if is_show_progress { pb.enable_steady_tick(Duration::from_millis(300)); } + self.rule_keys = self.get_all_keys(&rule_files); let mut detection = detection::Detection::new(rule_files); let mut tl = Timeline::new(); @@ -2259,6 +2286,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -2431,6 +2460,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -2517,6 +2548,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -2602,6 +2635,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), @@ -2688,6 +2723,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), diff --git a/src/options/htmlreport.rs b/src/options/htmlreport.rs index 141e14a32..cf0c29c80 100644 --- a/src/options/htmlreport.rs +++ b/src/options/htmlreport.rs @@ -212,7 +212,7 @@ mod tests { "- Test rules: 783 (26.70%)".to_string(), "- Hayabusa rules: 138".to_string(), "- Sigma rules: 2,795".to_string(), - "- Total enabled detection rules: 2,933".to_string(), + "- Total detection rules: 2,933".to_string(), "- Elapsed time: 00:00:29.035".to_string(), "".to_string(), ]); @@ -303,6 +303,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -371,6 +373,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -442,6 +446,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, jsonl_timeline: false, geo_ip: None, @@ -510,6 +516,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, jsonl_timeline: false, geo_ip: None, @@ -533,7 +541,7 @@ mod tests { "- Test rules: 783 (26.70%)".to_string(), "- Hayabusa rules: 138".to_string(), "- Sigma rules: 2795".to_string(), - "- Total enabled detection rules: 2933".to_string(), + "- Total detection rules: 2933".to_string(), "- Elapsed time: 00:00:29.035".to_string(), "".to_string(), ]); @@ -587,7 +595,7 @@ mod tests { "- Test rules: 783 (26.70%)".to_string(), "- Hayabusa rules: 138".to_string(), "- Sigma rules: 2795".to_string(), - "- Total enabled detection rules: 2933".to_string(), + "- Total detection rules: 2933".to_string(), "- Elapsed time: 00:00:29.035".to_string(), "".to_string(), ]); diff --git a/src/options/profile.rs b/src/options/profile.rs index 0e71bf1f1..2375d8b8b 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -538,6 +538,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -617,6 +619,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, @@ -726,6 +730,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None, diff --git a/src/yaml.rs b/src/yaml.rs index 759f65533..0090be10b 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -767,6 +767,8 @@ mod tests { no_wizard: true, include_status: None, low_memory_mode: false, + enable_all_rules: false, + scan_all_evtx_files: false, }, geo_ip: None, output: None,