Support log-broker with redis (#642)

- Replace `tracing` to `log-broker`
- Receive and save log from redis to op_log_store every `fetch_interval`
- Redis client id should be `agent_id` format (giganto@localhost)

Close: #623

CHANGELOG.md

@@ -27,6 +27,9 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 - Added `request_from_peer: Option<bool>` argument to GraphQL endpoints:
   `netflow5_raw_events`, `netflow9_raw_events`, `secu_log_raw_events`,
   `statistics`.
+- Supported `log-broker` to send/receive operation log with redis server.
+  - Set the redis server with `redis_log_address`, `redis_log_agent_id` and
+    `redis_log_fetch_interval` in configuration options.
 
 ### Changed
 
@@ -40,6 +43,7 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 - Changed `export` GraphQL query's response value format from `{export_path}` to
 `{export_path}@{giganto_node_name}`
 - Changed `PEER_VERSION_REQ` to ">=0.16.0-alpha.1,<0.17.0"
+- Changed logging from `tracing` to `log-broker`.
 
 ### Fixed
 

Cargo.toml

@@ -28,12 +28,14 @@ graphql_client = "0.13"
 humantime = "2.1"
 humantime-serde = "1"
 libc = "0.2"
+log-broker = { git = "https://github.com/aicers/log-broker.git", tag = "0.1.0" }
 num_enum = "0.7"
 num-traits = "0.2"
 pcap = "1"
 proc-macro2 = "1.0"
 quinn = "0.10"
 quote = "1.0"
+redis = { version = "0.24", features = ["tokio-comp"]}
 reqwest = { version = "0.11", features = ["rustls-tls", "json"] }
 rocksdb = "0.21"
 roxy = { git = "https://github.com/aicers/roxy.git", tag = "0.2.1" }
@@ -46,7 +48,6 @@ syn = "2.0"
 tempfile = "3"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 toml_edit = "0.21"
-tracing = "0.1"
 warp = { version = "0.3", features = ["tls"] }
 x509-parser = "0.15"
 

README.md

@@ -36,6 +36,9 @@ roots = ["ca1.pem", "ca2.pem", "ca3.pem"]  # paths to CA certificate files
 ingest_address = "0.0.0.0:38370"           # address to listen for ingest QUIC
 publish_address = "0.0.0.0:38371"          # address to listen for publish QUIC
 graphql_address = "127.0.0.1:8443"         # giganto's graphql address
+redis_log_address = "127.0.0.1:6379"       # address to redis server
+redis_log_agent_id = "giganto@localhost"   # agent id to send log to redis
+redis_log_fetch_interval = "10m"           # log fetch interval from redis
 data_dir = "tests/data"                    # path to directory to store data
 retention = "100d"                         # retention period for data
 log_dir = "/data/logs/apps"                # path to giganto's syslog file

src/graphql.rs

@@ -29,6 +29,7 @@ use chrono::{DateTime, TimeZone, Utc};
 use giganto_client::ingest::Packet as pk;
 use graphql_client::Response as GraphQlResponse;
 use libc::timeval;
+use log_broker::{error, LogLocation};
 use num_traits::AsPrimitive;
 use pcap::{Capture, Linktype, Packet, PacketHeader};
 use serde::Deserialize;
@@ -48,7 +49,6 @@ use std::{
 };
 use tempfile::tempfile;
 use tokio::sync::Notify;
-use tracing::error;
 
 pub const TIMESTAMP_SIZE: usize = 8;
 
@@ -412,6 +412,7 @@ where
         if records.len() == size {
             if invalid_data_cnt > 1 {
                 error!(
+                    LogLocation::Both,
                     "failed to read database or invalid data of {data_type} #{invalid_data_cnt}"
                 );
             }