Skip to content

Commit

Permalink
Implement Drain3-based log clustering flow (#23)
Browse files Browse the repository at this point in the history
- Drain3 
- Ray-based experiment
  • Loading branch information
Liangshumin authored Sep 17, 2022
1 parent bba0a2f commit 5872919
Show file tree
Hide file tree
Showing 12 changed files with 1,365 additions and 0 deletions.
2 changes: 2 additions & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,7 @@ header:
- '**/*.lock'
- '**/*.csv'
- '**/*.log'
- 'experiments/log/clustering/drain_parser/*'
- '**/*.ini'

comment: on-failure
28 changes: 28 additions & 0 deletions experiments/log/clustering/drain3.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[SNAPSHOT]
snapshot_interval_minutes = 10
compress_state = True

[MASKING]
masking = [
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)(([0-9a-f]{2,}:){3,}([0-9a-f]{2,}))((?=[^A-Za-z0-9])|$)", "mask_with": "ID"},
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})((?=[^A-Za-z0-9])|$)", "mask_with": "IP"},
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)([0-9a-f]{6,} ?){3,}((?=[^A-Za-z0-9])|$)", "mask_with": "SEQ"},
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)([0-9A-F]{4} ?){4,}((?=[^A-Za-z0-9])|$)", "mask_with": "SEQ"},
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)(0x[a-f0-9A-F]+)((?=[^A-Za-z0-9])|$)", "mask_with": "HEX"},
{"regex_pattern":"((?<=[^A-Za-z0-9])|^)([\\-\\+]?\\d+)((?=[^A-Za-z0-9])|$)", "mask_with": "NUM"},
{"regex_pattern":"(?<=executed cmd )(\".+?\")", "mask_with": "CMD"}
]
mask_prefix = <:
mask_suffix = :>

[DRAIN]
sim_th = 0.4
depth = 4
max_children = 100
max_clusters = 1024
max_logs=10240
extra_delimiters = ["_"]

[PROFILING]
enabled = True
report_sec = 30
Loading

0 comments on commit 5872919

Please sign in to comment.