From 3c935b50c65969ea9f0cac833dfacbc68b006121 Mon Sep 17 00:00:00 2001 From: Alexander Tesfamichael Date: Mon, 9 Oct 2023 15:46:49 +0200 Subject: [PATCH] feat(inclusion_monitor): add payload metadata Adds details of the missed payload to the alert using our logs. --- Cargo.lock | 11 +- Cargo.toml | 1 + src/phoenix.rs | 6 +- src/phoenix/env.rs | 29 +- src/phoenix/inclusion_monitor.rs | 51 ++- src/phoenix/inclusion_monitor/loki_client.rs | 134 +++++++ .../test_data/payload_logs_7496729.json | 350 ++++++++++++++++++ 7 files changed, 554 insertions(+), 28 deletions(-) create mode 100644 src/phoenix/inclusion_monitor/loki_client.rs create mode 100644 src/phoenix/inclusion_monitor/test_data/payload_logs_7496729.json diff --git a/Cargo.lock b/Cargo.lock index 649ce32..2102a7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -890,6 +890,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" + [[package]] name = "instant" version = "0.1.12" @@ -1232,9 +1238,9 @@ checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" @@ -1406,6 +1412,7 @@ dependencies = [ "futures", "gcp-bigquery-client", "hex", + "indoc", "itertools", "lazy_static", "rand", diff --git a/Cargo.toml b/Cargo.toml index 7583886..baa0bcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ flate2 = "1.0.25" futures = "0.3.25" gcp-bigquery-client = { git = "https://github.com/blombern/gcp-bigquery-client.git" } hex = "0.4.3" +indoc = "2.0.4" itertools = "0.10.5" lazy_static = "1" rand = "0.8.5" diff --git a/src/phoenix.rs b/src/phoenix.rs index d3204bd..56772d7 100644 --- a/src/phoenix.rs +++ b/src/phoenix.rs @@ -29,7 +29,8 @@ use crate::phoenix::{ }; use self::{ - demotion_monitor::run_demotion_monitor, inclusion_monitor::run_inclusion_monitor, + demotion_monitor::run_demotion_monitor, + inclusion_monitor::{run_inclusion_monitor, LokiClient}, promotion_monitor::run_promotion_monitor, }; @@ -220,11 +221,12 @@ async fn run_ops_monitors() -> Result<()> { &max_retry_duration, ) .await?; + let loki_client = LokiClient::new(APP_CONFIG.loki_url.clone()); loop { let canonical_horizon = Utc::now() - Duration::minutes(APP_CONFIG.canonical_wait_minutes); run_demotion_monitor(&relay_pool, &mev_pool).await?; - run_inclusion_monitor(&relay_pool, &mev_pool, &canonical_horizon).await?; + run_inclusion_monitor(&relay_pool, &mev_pool, &canonical_horizon, &loki_client).await?; run_promotion_monitor(&relay_pool, &mev_pool, &canonical_horizon).await?; tokio::time::sleep(Duration::minutes(1).to_std().unwrap()).await; } diff --git a/src/phoenix/env.rs b/src/phoenix/env.rs index 469ecef..60c5789 100644 --- a/src/phoenix/env.rs +++ b/src/phoenix/env.rs @@ -6,25 +6,26 @@ use crate::env::{deserialize_urls, get_app_config, Env}; #[derive(Deserialize)] pub struct AppConfig { - pub env: Env, - pub port: u16, - pub database_url: String, - pub relay_database_url: String, - #[serde(deserialize_with = "deserialize_urls")] - pub consensus_nodes: Vec, - #[serde(deserialize_with = "deserialize_urls")] - pub validation_nodes: Vec, - pub opsgenie_api_key: String, - pub telegram_api_key: String, - pub telegram_channel_id: String, #[serde(default = "default_wait")] pub canonical_wait_minutes: i64, - /// Slot range to check for counting missed slots - #[serde(default = "default_missed_slots_range")] - pub missed_slots_check_range: i64, + #[serde(deserialize_with = "deserialize_urls")] + pub consensus_nodes: Vec, + pub database_url: String, + pub env: Env, + pub loki_url: String, /// Minimum number of missed slots per check interval to trigger an alert #[serde(default = "default_missed_slots_alert_threshold")] pub missed_slots_alert_threshold: i64, + /// Slot range to check for counting missed slots + #[serde(default = "default_missed_slots_range")] + pub missed_slots_check_range: i64, + pub opsgenie_api_key: String, + pub port: u16, + pub relay_database_url: String, + pub telegram_api_key: String, + pub telegram_channel_id: String, + #[serde(deserialize_with = "deserialize_urls")] + pub validation_nodes: Vec, } fn default_wait() -> i64 { diff --git a/src/phoenix/inclusion_monitor.rs b/src/phoenix/inclusion_monitor.rs index 846c397..351458a 100644 --- a/src/phoenix/inclusion_monitor.rs +++ b/src/phoenix/inclusion_monitor.rs @@ -1,9 +1,15 @@ -use anyhow::Result; +mod loki_client; + +pub use loki_client::LokiClient; + use chrono::{DateTime, TimeZone, Utc}; +use indoc::formatdoc; use reqwest::StatusCode; use sqlx::{PgPool, Row}; use tracing::{error, info, warn}; +use loki_client::PayloadLogStats; + use crate::{ beacon_api::BeaconApi, env::{ToBeaconExplorerUrl, ToNetwork}, @@ -27,7 +33,7 @@ async fn get_delivered_payloads( relay_pool: &PgPool, start: &DateTime, end: &DateTime, -) -> Result> { +) -> anyhow::Result> { let query = format!( " SELECT @@ -64,7 +70,7 @@ async fn insert_missed_slot( slot_number: &i64, relayed: &String, canonical: Option<&String>, -) -> Result<()> { +) -> anyhow::Result<()> { sqlx::query!( r#" INSERT INTO missed_slots (slot_number, relayed_block_hash, canonical_block_hash) @@ -96,7 +102,8 @@ pub async fn run_inclusion_monitor( relay_pool: &PgPool, mev_pool: &PgPool, canonical_horizon: &DateTime, -) -> Result<()> { + log_client: &LokiClient, +) -> anyhow::Result<()> { let beacon_api = BeaconApi::new(&APP_CONFIG.consensus_nodes); let checkpoint = match checkpoint::get_checkpoint(mev_pool, CheckpointId::Inclusion).await? { @@ -152,12 +159,36 @@ pub async fn run_inclusion_monitor( insert_missed_slot(mev_pool, &payload.slot, &payload.block_hash, None).await?; - alert::send_telegram_alert(&format!( - "delivered block not found for slot [{slot}]({url}/slot/{slot})", - slot = payload.slot, - url = explorer_url, - )) - .await?; + let PayloadLogStats { + pre_publish_duration_ms, + publish_duration_ms, + received_at_slot_age_ms, + request_download_duration_ms, + } = log_client.payload_logs(&(payload.slot as i32)).await?; + + let publish_took_too_long = publish_duration_ms > 1000; + let request_arrived_too_late = request_download_duration_ms > 1000; + let safe_to_ignore = request_arrived_too_late && !publish_took_too_long; + + let msg = formatdoc!( + " + delivered block not found for slot + + [beaconcha\\.in/slot/{slot}]({explorer_url}/slot/{slot}) + + ``` + pre_publish_duration_ms: {pre_publish_duration_ms} + publish_duration_ms: {publish_duration_ms} + received_at_slot_age_ms: {received_at_slot_age_ms} + request_download_duration_ms: {request_download_duration_ms} + safe_to_ignore: {safe_to_ignore} + slot: {slot} + ``` + ", + slot = payload.slot + ); + + alert::send_telegram_alert(&msg).await?; } else { error!( "error getting block hash for slot {}: {}", diff --git a/src/phoenix/inclusion_monitor/loki_client.rs b/src/phoenix/inclusion_monitor/loki_client.rs new file mode 100644 index 0000000..1f93497 --- /dev/null +++ b/src/phoenix/inclusion_monitor/loki_client.rs @@ -0,0 +1,134 @@ +use std::str::FromStr; + +use anyhow::Context; +use chrono::{DateTime, TimeZone, Utc}; +use reqwest::Url; + +/// Statistics on payloads requested. Used to determine if a payload which failed to make it +/// on-chain should concern us. +#[derive(Debug)] +pub struct PayloadLogStats { + pub pre_publish_duration_ms: i64, + // The time it took to call our consensus node and have it publish the block. + pub publish_duration_ms: i64, + pub received_at_slot_age_ms: i64, + pub request_download_duration_ms: i64, +} + +fn date_time_from_timestamp( + request_finished_log: &serde_json::Value, + key: &str, +) -> anyhow::Result> { + request_finished_log[key] + .as_str() + .and_then(|timestamp| timestamp.parse::().ok()) + .and_then(|timestamp| Utc.timestamp_millis_opt(timestamp).single()) + .with_context(|| format!("failed to parse {key} as timestamp from payload log")) +} + +impl FromStr for PayloadLogStats { + type Err = anyhow::Error; + + fn from_str(text: &str) -> Result { + let request_finished_log: serde_json::Value = { + let log_data: serde_json::Value = serde_json::from_str(&text) + .context("failed to parse payload log request body as JSON")?; + + // This is the array of parsed log lines and their raw values. + let results = log_data["data"]["result"] + .as_array() + .context("expected at least one log line in payload logs response")?; + + results + .iter() + .find(|result| { + let stream = &result["stream"]; + let msg = stream["msg"].as_str().unwrap_or(""); + msg.contains("request finished") + }) + .map(|result| &result["stream"]) + .cloned() + .with_context(|| format!("no proposer-api log lines with msg field found"))? + }; + + let received_at = date_time_from_timestamp(&request_finished_log, "timestampRequestStart")?; + let decoded_at = date_time_from_timestamp(&request_finished_log, "timestampAfterDecode")?; + let pre_publish_at = + date_time_from_timestamp(&request_finished_log, "timestampBeforePublishing")?; + let post_publish_at = + date_time_from_timestamp(&request_finished_log, "timestampAfterPublishing")?; + let received_at_slot_age_ms = request_finished_log["msIntoSlot"] + .as_str() + .and_then(|s| s.parse::().ok()) + .context("failed to parse msIntoSlot as i64")?; + + let pre_publish_duration_ms = pre_publish_at + .signed_duration_since(received_at) + .num_milliseconds(); + + let publish_duration_ms = post_publish_at + .signed_duration_since(pre_publish_at) + .num_milliseconds(); + + let request_download_duration_ms = decoded_at + .signed_duration_since(received_at) + .num_milliseconds(); + + let payload_log_stats = PayloadLogStats { + pre_publish_duration_ms, + publish_duration_ms, + received_at_slot_age_ms, + request_download_duration_ms, + }; + + Ok(payload_log_stats) + } +} + +pub struct LokiClient { + client: reqwest::Client, + server_url: String, +} + +impl LokiClient { + pub fn new(server_url: String) -> Self { + Self { + client: reqwest::Client::new(), + server_url, + } + } + + pub async fn payload_logs(&self, slot: &i32) -> anyhow::Result { + let query = format!(r#"{{app="proposer-api"}} |= `"slot":{slot}` | json"#); + let since = "15m"; + + let url = format!("{}/loki/api/v1/query_range", self.server_url); + let url_with_params = + Url::parse_with_params(&url, &[("query", query.as_str()), ("since", since)])?; + + let response = self.client.get(url_with_params).send().await?; + let body = response.text().await?; + + body.parse::() + } +} + +#[cfg(test)] +mod tests { + use std::{fs::File, io::Read}; + + use super::*; + + #[test] + fn parse_log_response() { + let str = File::open("src/phoenix/inclusion_monitor/test_data/payload_logs_7496729.json") + .map(|mut file| { + let mut str = String::new(); + file.read_to_string(&mut str).unwrap(); + str + }) + .unwrap(); + + str.parse::().unwrap(); + } +} diff --git a/src/phoenix/inclusion_monitor/test_data/payload_logs_7496729.json b/src/phoenix/inclusion_monitor/test_data/payload_logs_7496729.json new file mode 100644 index 0000000..2eb7b54 --- /dev/null +++ b/src/phoenix/inclusion_monitor/test_data/payload_logs_7496729.json @@ -0,0 +1,350 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "proposer-api", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "container": "proposer-api", + "contentLength": "81587", + "feeRecipient": "0x4675c7e5baafbffbca748158becba61ef3b0a263", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "headSlot": "7496728", + "headSlotEpochPos": "25", + "idArg": "", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "method": "getPayload", + "mevBoostV": "v1.6", + "msIntoSlot": "3625", + "msg": "getPayload request received", + "namespace": "default", + "node_name": "kate", + "pod": "proposer-api-cf64dbf6-wmz9h", + "proposerIndex": "138873", + "proposerPubkey": "0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488", + "service": "relay/api", + "slot": "7496729", + "slotEpochPos": "26", + "slotStartSec": "1696784771", + "stream": "stdout", + "time": "2023-10-08T17:06:14.62824969Z", + "timestampAfterDecode": "1696784774625", + "timestampAfterSignatureVerify": "1696784774628", + "timestampRequestStart": "1696784773892", + "ua": "mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336", + "version": "" + }, + "values": [ + [ + "1696784774628249690", + "{\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"contentLength\":81587,\"feeRecipient\":\"0x4675c7e5baafbffbca748158becba61ef3b0a263\",\"headSlot\":7496728,\"headSlotEpochPos\":25,\"idArg\":\"\",\"level\":\"info\",\"method\":\"getPayload\",\"mevBoostV\":\"v1.6\",\"msIntoSlot\":3625,\"msg\":\"getPayload request received\",\"proposerIndex\":138873,\"proposerPubkey\":\"0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488\",\"service\":\"relay/api\",\"slot\":7496729,\"slotEpochPos\":26,\"slotStartSec\":1696784771,\"time\":\"2023-10-08T17:06:14.62824969Z\",\"timestampAfterDecode\":1696784774625,\"timestampAfterSignatureVerify\":1696784774628,\"timestampRequestStart\":1696784773892,\"ua\":\"mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336\",\"version\":\"\"}" + ] + ] + }, + { + "stream": { + "app": "proposer-api", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "container": "proposer-api", + "contentLength": "81587", + "feeRecipient": "0x4675c7e5baafbffbca748158becba61ef3b0a263", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "headSlot": "7496728", + "headSlotEpochPos": "25", + "idArg": "", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "method": "getPayload", + "mevBoostV": "v1.6", + "msIntoSlot": "3625", + "msNeededForPublishing": "647", + "msg": "block published through beacon node", + "namespace": "default", + "node_name": "kate", + "pod": "proposer-api-cf64dbf6-wmz9h", + "proposerIndex": "138873", + "proposerPubkey": "0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488", + "service": "relay/api", + "slot": "7496729", + "slotEpochPos": "26", + "slotStartSec": "1696784771", + "stream": "stdout", + "time": "2023-10-08T17:06:15.285183899Z", + "timestampAfterAlreadyDeliveredCheck": "1696784774635", + "timestampAfterDecode": "1696784774625", + "timestampAfterLoadResponse": "1696784774634", + "timestampAfterPublishing": "1696784775285", + "timestampAfterSignatureVerify": "1696784774628", + "timestampBeforePublishing": "1696784774638", + "timestampRequestStart": "1696784773892", + "ua": "mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336", + "version": "" + }, + "values": [ + [ + "1696784775285183899", + "{\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"contentLength\":81587,\"feeRecipient\":\"0x4675c7e5baafbffbca748158becba61ef3b0a263\",\"headSlot\":7496728,\"headSlotEpochPos\":25,\"idArg\":\"\",\"level\":\"info\",\"method\":\"getPayload\",\"mevBoostV\":\"v1.6\",\"msIntoSlot\":3625,\"msNeededForPublishing\":647,\"msg\":\"block published through beacon node\",\"proposerIndex\":138873,\"proposerPubkey\":\"0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488\",\"service\":\"relay/api\",\"slot\":7496729,\"slotEpochPos\":26,\"slotStartSec\":1696784771,\"time\":\"2023-10-08T17:06:15.285183899Z\",\"timestampAfterAlreadyDeliveredCheck\":1696784774635,\"timestampAfterDecode\":1696784774625,\"timestampAfterLoadResponse\":1696784774634,\"timestampAfterPublishing\":1696784775285,\"timestampAfterSignatureVerify\":1696784774628,\"timestampBeforePublishing\":1696784774638,\"timestampRequestStart\":1696784773892,\"ua\":\"mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336\",\"version\":\"\"}" + ] + ] + }, + { + "stream": { + "app": "proposer-api", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "blockNumber": "18307137", + "container": "proposer-api", + "contentLength": "81587", + "feeRecipient": "0x4675c7e5baafbffbca748158becba61ef3b0a263", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "headSlot": "7496728", + "headSlotEpochPos": "25", + "idArg": "", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "method": "getPayload", + "mevBoostV": "v1.6", + "msIntoSlot": "3625", + "msg": "request finished", + "namespace": "default", + "node_name": "kate", + "numTx": "196", + "pod": "proposer-api-cf64dbf6-wmz9h", + "proposerIndex": "138873", + "proposerPubkey": "0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488", + "requestDurationMs": "2425", + "service": "relay/api", + "slot": "7496729", + "slotEpochPos": "26", + "slotStartSec": "1696784771", + "stream": "stdout", + "time": "2023-10-08T17:06:16.318189593Z", + "timestampAfterAlreadyDeliveredCheck": "1696784774635", + "timestampAfterDecode": "1696784774625", + "timestampAfterLoadResponse": "1696784774634", + "timestampAfterPublishing": "1696784775285", + "timestampAfterSignatureVerify": "1696784774628", + "timestampBeforePublishing": "1696784774638", + "timestampRequestFin": "1696784776318", + "timestampRequestStart": "1696784773892", + "ua": "mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336", + "version": "" + }, + "values": [ + [ + "1696784776318189593", + "{\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"blockNumber\":18307137,\"contentLength\":81587,\"feeRecipient\":\"0x4675c7e5baafbffbca748158becba61ef3b0a263\",\"headSlot\":7496728,\"headSlotEpochPos\":25,\"idArg\":\"\",\"level\":\"info\",\"method\":\"getPayload\",\"mevBoostV\":\"v1.6\",\"msIntoSlot\":3625,\"msg\":\"request finished\",\"numTx\":196,\"proposerIndex\":138873,\"proposerPubkey\":\"0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488\",\"requestDurationMs\":2425,\"service\":\"relay/api\",\"slot\":7496729,\"slotEpochPos\":26,\"slotStartSec\":1696784771,\"time\":\"2023-10-08T17:06:16.318189593Z\",\"timestampAfterAlreadyDeliveredCheck\":1696784774635,\"timestampAfterDecode\":1696784774625,\"timestampAfterLoadResponse\":1696784774634,\"timestampAfterPublishing\":1696784775285,\"timestampAfterSignatureVerify\":1696784774628,\"timestampBeforePublishing\":1696784774638,\"timestampRequestFin\":1696784776318,\"timestampRequestStart\":1696784773892,\"ua\":\"mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336\",\"version\":\"\"}" + ] + ] + }, + { + "stream": { + "app": "proposer-api", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "blockNumber": "18307137", + "container": "proposer-api", + "contentLength": "81587", + "feeRecipient": "0x4675c7e5baafbffbca748158becba61ef3b0a263", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "headSlot": "7496728", + "headSlotEpochPos": "25", + "idArg": "", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "method": "getPayload", + "mevBoostV": "v1.6", + "msIntoSlot": "3625", + "msg": "no demotion in getPayload, successful block proposal", + "namespace": "default", + "node_name": "kate", + "numTx": "196", + "pod": "proposer-api-cf64dbf6-wmz9h", + "proposerIndex": "138873", + "proposerPubkey": "0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488", + "service": "relay/api", + "slot": "7496729", + "slotEpochPos": "26", + "slotStartSec": "1696784771", + "stream": "stdout", + "time": "2023-10-08T17:06:16.321953648Z", + "timestampAfterAlreadyDeliveredCheck": "1696784774635", + "timestampAfterDecode": "1696784774625", + "timestampAfterLoadResponse": "1696784774634", + "timestampAfterPublishing": "1696784775285", + "timestampAfterSignatureVerify": "1696784774628", + "timestampBeforePublishing": "1696784774638", + "timestampRequestStart": "1696784773892", + "ua": "mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336", + "version": "" + }, + "values": [ + [ + "1696784776321953648", + "{\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"blockNumber\":18307137,\"contentLength\":81587,\"feeRecipient\":\"0x4675c7e5baafbffbca748158becba61ef3b0a263\",\"headSlot\":7496728,\"headSlotEpochPos\":25,\"idArg\":\"\",\"level\":\"info\",\"method\":\"getPayload\",\"mevBoostV\":\"v1.6\",\"msIntoSlot\":3625,\"msg\":\"no demotion in getPayload, successful block proposal\",\"numTx\":196,\"proposerIndex\":138873,\"proposerPubkey\":\"0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488\",\"service\":\"relay/api\",\"slot\":7496729,\"slotEpochPos\":26,\"slotStartSec\":1696784771,\"time\":\"2023-10-08T17:06:16.321953648Z\",\"timestampAfterAlreadyDeliveredCheck\":1696784774635,\"timestampAfterDecode\":1696784774625,\"timestampAfterLoadResponse\":1696784774634,\"timestampAfterPublishing\":1696784775285,\"timestampAfterSignatureVerify\":1696784774628,\"timestampBeforePublishing\":1696784774638,\"timestampRequestStart\":1696784773892,\"ua\":\"mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336\",\"version\":\"\"}" + ] + ] + }, + { + "stream": { + "app": "proposer-api", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "blockNumber": "18307137", + "container": "proposer-api", + "contentLength": "81587", + "feeRecipient": "0x4675c7e5baafbffbca748158becba61ef3b0a263", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "headSlot": "7496728", + "headSlotEpochPos": "25", + "idArg": "", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "method": "getPayload", + "metric_stream": "execution-payload-delivered", + "mevBoostV": "v1.6", + "msIntoSlot": "3625", + "msg": "execution payload delivered", + "namespace": "default", + "node_name": "kate", + "numTx": "196", + "pod": "proposer-api-cf64dbf6-wmz9h", + "proposerIndex": "138873", + "proposerPubkey": "0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488", + "service": "relay/api", + "slot": "7496729", + "slotEpochPos": "26", + "slotStartSec": "1696784771", + "stream": "stdout", + "time": "2023-10-08T17:06:16.310283235Z", + "timestampAfterAlreadyDeliveredCheck": "1696784774635", + "timestampAfterDecode": "1696784774625", + "timestampAfterLoadResponse": "1696784774634", + "timestampAfterPublishing": "1696784775285", + "timestampAfterSignatureVerify": "1696784774628", + "timestampBeforePublishing": "1696784774638", + "timestampRequestStart": "1696784773892", + "ua": "mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336", + "version": "" + }, + "values": [ + [ + "1696784776310283235", + "{\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"blockNumber\":18307137,\"contentLength\":81587,\"feeRecipient\":\"0x4675c7e5baafbffbca748158becba61ef3b0a263\",\"headSlot\":7496728,\"headSlotEpochPos\":25,\"idArg\":\"\",\"level\":\"info\",\"method\":\"getPayload\",\"mevBoostV\":\"v1.6\",\"msIntoSlot\":3625,\"msg\":\"execution payload delivered\",\"numTx\":196,\"proposerIndex\":138873,\"proposerPubkey\":\"0x8f94956dde1ba8d8a800163e0c654553a5e05eabebed8d443ffc5f52a77ffffc5ec69a14e406b4982371ddc96c007488\",\"service\":\"relay/api\",\"slot\":7496729,\"slotEpochPos\":26,\"slotStartSec\":1696784771,\"time\":\"2023-10-08T17:06:16.310283235Z\",\"timestampAfterAlreadyDeliveredCheck\":1696784774635,\"timestampAfterDecode\":1696784774625,\"timestampAfterLoadResponse\":1696784774634,\"timestampAfterPublishing\":1696784775285,\"timestampAfterSignatureVerify\":1696784774628,\"timestampBeforePublishing\":1696784774638,\"timestampRequestStart\":1696784773892,\"ua\":\"mev-boost/v1.6 Lighthouse/v4.3.0-dfcb336\",\"version\":\"\"}" + ] + ] + }, + { + "stream": { + "app": "proposer-api", + "beacon": "http://10.0.0.1:3500", + "blockHash": "0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096", + "component": "beaconClient", + "container": "proposer-api", + "filename": "/var/log/pods/default_proposer-api-cf64dbf6-wmz9h_5805ffc1-3abf-4e10-9468-1120c8add007/proposer-api/0.log", + "job": "default/proposer-api", + "level": "info", + "level_extracted": "info", + "msg": "published block", + "namespace": "default", + "node_name": "kate", + "pod": "proposer-api-cf64dbf6-wmz9h", + "service": "relay/api", + "slot": "7496729", + "statusCode": "200", + "stream": "stdout", + "time": "2023-10-08T17:06:15.285144063Z", + "version": "" + }, + "values": [ + [ + "1696784775285144063", + "{\"beacon\":\"http://10.0.0.1:3500\",\"blockHash\":\"0x73bec1f6e7571247af2037120680a8a7e57fdbfc1d27202d619638141f0e0096\",\"component\":\"beaconClient\",\"level\":\"info\",\"msg\":\"published block\",\"service\":\"relay/api\",\"slot\":7496729,\"statusCode\":200,\"time\":\"2023-10-08T17:06:15.285144063Z\",\"version\":\"\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 230885010, + "linesProcessedPerSecond": 785906, + "totalBytesProcessed": 67432281, + "totalLinesProcessed": 229532, + "execTime": 0.29206, + "queueTime": 0.000567, + "subqueries": 0, + "totalEntriesReturned": 6, + "splits": 97, + "shards": 3 + }, + "querier": { + "store": { + "totalChunksRef": 48, + "totalChunksDownloaded": 48, + "chunksDownloadTime": 67631518, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 67432281, + "decompressedLines": 229532, + "compressedBytes": 5421475, + "totalDuplicates": 2 + } + } + }, + "ingester": { + "totalReached": 6, + "totalChunksMatched": 0, + "totalBatches": 0, + "totalLinesSent": 0, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0 + } + } + }, + "cache": { + "chunk": { + "entriesFound": 44, + "entriesRequested": 48, + "entriesStored": 4, + "bytesReceived": 16970760, + "bytesSent": 244499, + "requests": 3, + "downloadTime": 28564786 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0 + }, + "result": { + "entriesFound": 94, + "entriesRequested": 95, + "entriesStored": 0, + "bytesReceived": 9218, + "bytesSent": 0, + "requests": 95, + "downloadTime": 101725232 + } + } + } + } +}