feat: Add Lexer and LogParser for tokenizing log events and struc…

…ture parsed tokens from a given log stream. (#11)

.gitignore

@@ -1,4 +1,5 @@
 # Build dir
+/examples/target
 /target
 
 # Dev env configs

Cargo.toml

@@ -5,3 +5,4 @@ edition = "2021"
 
 [dependencies]
 regex-syntax = "0.8.5"
+serde_yaml = "0.9.34"

examples/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "sample_program"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+log-surgeon = { path = ".." }

examples/logs/simple.log

@@ -0,0 +1,8 @@
+This log event doesn't have a timestamp
+TIMESTAMP Id: 3190; This is a
+multi-line log event with unicode: 这是一个有多行的日志
+TIMESTAMP Id: 0; This is a multi-line log event. I will pay
+you 1000 dollars to test this file.
+TIMESTAMP Id: 0; This is a variable=0
+TIMESTAMP Id: 0; But this is:0
+TIMESTAMP Variable with delimiter: a b a b a a a a

examples/schema.yaml

@@ -0,0 +1,15 @@
+timestamp:
+  # E.g. 2015-01-31T15:50:45.392
+  - '\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}'
+  # E.g. 2015-01-31T15:50:45,392
+  - '\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2},\d{3}'
+  # E.g. 2015-01-31 15:50:45
+  - '\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}'
+
+delimiters: " \t\r\n:,!;%"
+
+variables:
+  int: '\-{0,1}\d+'
+  float: '\-{0,1}[0-9]+\.[0-9]+'
+  hex: '(0x){0,1}([0-9a-f]+)|([0-9A-F]+)'
+  loglevel: '(INFO)|(DEBUG)|(WARN)|(ERROR)|(TRACE)|(FATAL)'

examples/schema_simple.yaml

@@ -0,0 +1,8 @@
+timestamp:
+  - 'TIMESTAMP'
+
+delimiters: " \t\r\n:,!;%"
+
+variables:
+  int: '\-{0,1}\d+'
+  with_delimiter: 'a a'

examples/src/main.rs

@@ -0,0 +1,24 @@
+use log_surgeon::error_handling::Result;
+use log_surgeon::parser::SchemaConfig;
+use log_surgeon::log_parser::LogEvent;
+use log_surgeon::log_parser::LogParser;
+
+use std::rc::Rc;
+
+fn main() -> Result<()> {
+    let project_root = env!("CARGO_MANIFEST_DIR");
+    let schema_path = std::path::Path::new(project_root).join("schema_simple.yaml");
+    let log_path = std::path::Path::new(project_root)
+        .join("logs")
+        .join("simple.log");
+
+    let parsed_schema = Rc::new(SchemaConfig::parse_from_file(schema_path.to_str().unwrap())?);
+    let mut log_parser = LogParser::new(parsed_schema.clone())?;
+    log_parser.set_input_file(log_path.to_str().unwrap())?;
+
+    while let Some(log_event) = log_parser.parse_next_log_event()? {
+        println!("{:?}", log_event);
+    }
+
+    Ok(())
+}