From f42fe638d9c59332d33c6b4c2dfa6b45cf7916ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Tomi=C4=87?= Date: Fri, 6 Dec 2024 23:08:29 +0100 Subject: [PATCH] chore(node): Remove legacy mercury.dfinity.systems logging targets (#2740) NODE-1538 The mainnet does not push logs, and *.mercury.dfinity.systems machines have been decommissioned over a year ago. We pull logs from the mainnet machines, using vector and a [log-fetcher source/adapter that DRE team wrote](https://github.com/dfinity/dre/tree/main/rs/log-fetcher). So it's really worth removing these URLs from the codebase since they only cause spam entries in the system logs. This PR also updates update-config.service to *no longer* use filebeat.conf as an input to update the config. Instead, it sets Logging to the default to empty values. After we switch to [USE the new config](https://dfinity.atlassian.net/browse/NODE-1477), the [filebeat exec condition will fail](https://github.com/dfinity/ic/blob/d7249e0cf1981b0727d2657182046edaefa8c0b5/ic-os/components/monitoring/filebeat/filebeat.service#L22), and filebeat will not do any work. --------- Co-authored-by: Andrew Battat --- .../dev-generate-guestos-config.sh | 1 - .../generate-guestos-config.sh | 1 - ic-os/setupos/data/deployment.json.template | 2 +- rs/ic_os/config/src/deployment_json.rs | 39 ++++++++++ .../config/src/generate_testnet_config.rs | 2 +- rs/ic_os/config/src/lib.rs | 19 +---- rs/ic_os/config/src/main.rs | 7 +- rs/ic_os/config/src/update_config.rs | 77 +------------------ rs/ic_os/config_types/src/lib.rs | 9 +-- 9 files changed, 50 insertions(+), 107 deletions(-) diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh index 024fc730284..d1de73b5a0c 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/dev-generate-guestos-config.sh @@ -83,7 +83,6 @@ function read_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then diff --git a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh index 3d125938234..86f038a0b78 100755 --- a/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh +++ b/ic-os/components/hostos-scripts/generate-guestos-config/generate-guestos-config.sh @@ -83,7 +83,6 @@ function read_variables() { function assemble_config_media() { cmd=(/opt/ic/bin/build-bootstrap-config-image.sh ${MEDIA}) cmd+=(--nns_public_key "/boot/config/nns_public_key.pem") - cmd+=(--elasticsearch_hosts "$(/opt/ic/bin/fetch-property.sh --key=.logging.hosts --metric=hostos_logging_hosts --config=${DEPLOYMENT})") cmd+=(--ipv6_address "$(/opt/ic/bin/hostos_tool generate-ipv6-address --node-type GuestOS)") cmd+=(--ipv6_gateway "${ipv6_gateway}") if [[ -n "$ipv4_address" && -n "$ipv4_prefix_length" && -n "$ipv4_gateway" && -n "$domain" ]]; then diff --git a/ic-os/setupos/data/deployment.json.template b/ic-os/setupos/data/deployment.json.template index 3ff2bcfdab6..d3bb1638e44 100644 --- a/ic-os/setupos/data/deployment.json.template +++ b/ic-os/setupos/data/deployment.json.template @@ -4,7 +4,7 @@ "mgmt_mac": null }, "logging": { - "hosts": "elasticsearch-node-0.mercury.dfinity.systems:443 elasticsearch-node-1.mercury.dfinity.systems:443 elasticsearch-node-2.mercury.dfinity.systems:443 elasticsearch-node-3.mercury.dfinity.systems:443" + "hosts": "" }, "nns": { "url": "NNS_URL" diff --git a/rs/ic_os/config/src/deployment_json.rs b/rs/ic_os/config/src/deployment_json.rs index 94c411f4068..3a35f532aa1 100644 --- a/rs/ic_os/config/src/deployment_json.rs +++ b/rs/ic_os/config/src/deployment_json.rs @@ -320,6 +320,41 @@ mod test { } }); + const DEPLOYMENT_STR_NO_LOGGING_HOSTS: &str = r#"{ + "deployment": { + "name": "mainnet", + "mgmt_mac": null + }, + "logging": { + "hosts": "" + }, + "nns": { + "url": "https://wiki.internetcomputer.org/" + }, + "resources": { + "memory": "490", + "cpu": "kvm" + } + }"#; + + static DEPLOYMENT_STRUCT_NO_LOGGING_HOSTS: Lazy = + Lazy::new(|| DeploymentSettings { + deployment: Deployment { + name: "mainnet".to_string(), + mgmt_mac: None, + }, + logging: Logging { + hosts: Default::default(), + }, + nns: Nns { + url: vec![Url::parse("https://wiki.internetcomputer.org").unwrap()], + }, + resources: Resources { + memory: 490, + cpu: Some("kvm".to_string()), + }, + }); + #[test] fn deserialize_deployment() { let parsed_deployment = { serde_json::from_str(DEPLOYMENT_STR).unwrap() }; @@ -358,6 +393,10 @@ mod test { let parsed_deployment = { serde_json::from_value(DEPLOYMENT_VALUE.clone()).unwrap() }; assert_eq!(*DEPLOYMENT_STRUCT, parsed_deployment); + + let parsed_deployment = { serde_json::from_str(DEPLOYMENT_STR_NO_LOGGING_HOSTS).unwrap() }; + + assert_eq!(*DEPLOYMENT_STRUCT_NO_LOGGING_HOSTS, parsed_deployment); } #[test] diff --git a/rs/ic_os/config/src/generate_testnet_config.rs b/rs/ic_os/config/src/generate_testnet_config.rs index c569cce9995..d13c7627425 100644 --- a/rs/ic_os/config/src/generate_testnet_config.rs +++ b/rs/ic_os/config/src/generate_testnet_config.rs @@ -172,7 +172,7 @@ fn create_guestos_config(config: GenerateTestnetConfigArgs) -> Result Result<()> { // get deployment.json variables let deployment_json_settings = get_deployment_settings(&deployment_json_path)?; - let logging = Logging { - elasticsearch_hosts: deployment_json_settings.logging.hosts.to_string(), - elasticsearch_tags: None, - }; - let mgmt_mac = resolve_mgmt_mac(deployment_json_settings.deployment.mgmt_mac)?; let node_reward_type = node_reward_type.expect("Node reward type is required."); @@ -228,7 +223,7 @@ pub fn main() -> Result<()> { node_reward_type: Some(node_reward_type), mgmt_mac, deployment_environment: deployment_json_settings.deployment.name.parse()?, - logging, + logging: Logging::default(), use_nns_public_key, nns_urls: deployment_json_settings.nns.url.clone(), use_node_operator_private_key, diff --git a/rs/ic_os/config/src/update_config.rs b/rs/ic_os/config/src/update_config.rs index c5a18fce0b1..f11b485d5ea 100644 --- a/rs/ic_os/config/src/update_config.rs +++ b/rs/ic_os/config/src/update_config.rs @@ -34,7 +34,6 @@ pub fn update_guestos_config() -> Result<()> { let network_settings = network_config_result.network_settings; let hostname = network_config_result.hostname.clone(); - let logging = read_filebeat_conf(config_dir)?; let nns_urls = read_nns_conf(config_dir)?; let use_nns_public_key = state_root.join("nns_public_key.pem").exists(); @@ -49,7 +48,7 @@ pub fn update_guestos_config() -> Result<()> { node_reward_type: None, mgmt_mac, deployment_environment, - logging, + logging: Logging::default(), use_nns_public_key, nns_urls, use_node_operator_private_key, @@ -144,36 +143,6 @@ struct NetworkConfigResult { hostname: Option, } -fn read_filebeat_conf(config_dir: &Path) -> Result { - let filebeat_conf_path = config_dir.join("filebeat.conf"); - let conf_map = match read_conf_file(&filebeat_conf_path) { - Ok(map) => map, - Err(_) => { - // Set default values if filebeat.conf doesn't exist - return Ok(Logging { - elasticsearch_hosts: "elasticsearch-node-0.mercury.dfinity.systems:443 \ - elasticsearch-node-1.mercury.dfinity.systems:443 \ - elasticsearch-node-2.mercury.dfinity.systems:443 \ - elasticsearch-node-3.mercury.dfinity.systems:443" - .to_string(), - elasticsearch_tags: None, - }); - } - }; - - let elasticsearch_hosts = conf_map - .get("elasticsearch_hosts") - .cloned() - .unwrap_or_default(); - - let elasticsearch_tags = conf_map.get("elasticsearch_tags").cloned(); - - Ok(Logging { - elasticsearch_hosts, - elasticsearch_tags, - }) -} - fn read_nns_conf(config_dir: &Path) -> Result> { let nns_conf_path = config_dir.join("nns.conf"); let conf_map = match read_conf_file(&nns_conf_path) { @@ -318,11 +287,6 @@ pub fn update_hostos_config( let deployment_json_settings = get_deployment_settings(deployment_json_path)?; - let logging = Logging { - elasticsearch_hosts: deployment_json_settings.logging.hosts.to_string(), - elasticsearch_tags: None, - }; - let mgmt_mac = resolve_mgmt_mac(deployment_json_settings.deployment.mgmt_mac)?; let use_nns_public_key = Path::new("/boot/config/nns_public_key.pem").exists(); @@ -334,7 +298,7 @@ pub fn update_hostos_config( node_reward_type, mgmt_mac, deployment_environment: deployment_json_settings.deployment.name.parse()?, - logging, + logging: Logging::default(), use_nns_public_key, nns_urls: deployment_json_settings.nns.url.clone(), use_node_operator_private_key, @@ -463,43 +427,6 @@ mod tests { Ok(()) } - #[test] - fn test_read_filebeat_conf_existing_file() -> Result<()> { - let dir = tempdir()?; - let filebeat_conf_path = dir.path().join("filebeat.conf"); - let mut file = fs::File::create(&filebeat_conf_path)?; - writeln!(file, "elasticsearch_hosts=host1:9200,host2:9200")?; - writeln!(file, "elasticsearch_tags=tag1,tag2")?; - - let logging = read_filebeat_conf(dir.path())?; - - assert_eq!( - logging.elasticsearch_hosts, - "host1:9200,host2:9200".to_string() - ); - assert_eq!(logging.elasticsearch_tags, Some("tag1,tag2".to_string())); - - Ok(()) - } - - #[test] - fn test_read_filebeat_conf_missing_file() -> Result<()> { - let dir = tempdir()?; - let logging = read_filebeat_conf(dir.path())?; - - assert_eq!( - logging.elasticsearch_hosts, - "elasticsearch-node-0.mercury.dfinity.systems:443 \ - elasticsearch-node-1.mercury.dfinity.systems:443 \ - elasticsearch-node-2.mercury.dfinity.systems:443 \ - elasticsearch-node-3.mercury.dfinity.systems:443" - .to_string() - ); - assert_eq!(logging.elasticsearch_tags, None); - - Ok(()) - } - #[test] fn test_read_nns_conf_existing_file() -> Result<()> { let dir = tempdir()?; diff --git a/rs/ic_os/config_types/src/lib.rs b/rs/ic_os/config_types/src/lib.rs index d2355887c56..87417f42829 100644 --- a/rs/ic_os/config_types/src/lib.rs +++ b/rs/ic_os/config_types/src/lib.rs @@ -186,10 +186,10 @@ impl FromStr for DeploymentEnvironment { } } -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Default)] pub struct Logging { /// Space-separated lists of hosts to ship logs to. - pub elasticsearch_hosts: String, + pub elasticsearch_hosts: Option, /// Space-separated list of tags to apply to exported log records. pub elasticsearch_tags: Option, } @@ -250,10 +250,7 @@ mod tests { node_reward_type: Some(String::new()), mgmt_mac: "00:00:00:00:00:00".parse()?, deployment_environment: DeploymentEnvironment::Testnet, - logging: Logging { - elasticsearch_hosts: String::new(), - elasticsearch_tags: None, - }, + logging: Logging::default(), use_nns_public_key: false, nns_urls: vec![], use_node_operator_private_key: false,