diff --git a/Cargo.lock b/Cargo.lock index 067729e..110d3d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,7 +1034,7 @@ dependencies = [ "atomic 0.6.0", "pear", "serde", - "toml", + "toml 0.8.2", "uncased", "version_check", ] @@ -1768,6 +1768,12 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "718e8fae447df0c7e1ba7f5189829e63fd536945c8988d61444c19039f16b670" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.1.0" @@ -2499,6 +2505,7 @@ version = "0.1.0" dependencies = [ "csv", "log", + "maplit", "rocket", "rocket_cors", "serde_json", @@ -2924,7 +2931,11 @@ dependencies = [ name = "shared" version = "0.1.0" dependencies = [ + "csv", + "log", + "serde", "serde_json", + "toml 0.5.11", "types", ] @@ -3490,6 +3501,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.8.2" @@ -3628,7 +3648,6 @@ dependencies = [ name = "tracker" version = "0.1.0" dependencies = [ - "csv", "env_logger", "log", "shared", diff --git a/bin/tracker/Cargo.toml b/bin/tracker/Cargo.toml index 03403c0..cb1d4cc 100644 --- a/bin/tracker/Cargo.toml +++ b/bin/tracker/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -csv = "1.3.0" log = "0.4" env_logger = "0.10.1" subxt = "0.32.1" diff --git a/bin/tracker/src/main.rs b/bin/tracker/src/main.rs index fb76025..9231f0a 100644 --- a/bin/tracker/src/main.rs +++ b/bin/tracker/src/main.rs @@ -43,9 +43,7 @@ const LOG_TARGET: &str = "tracker"; -use csv::WriterBuilder; -use shared::{file_path, parachains, round_to}; -use std::fs::OpenOptions; +use shared::{parachains, round_to, write_consumption}; use subxt::{blocks::Block, utils::H256, OnlineClient, PolkadotConfig}; use types::{Parachain, Timestamp, WeightConsumption}; @@ -115,40 +113,6 @@ async fn note_new_block( Ok(()) } -fn write_consumption( - para: Parachain, - consumption: WeightConsumption, -) -> Result<(), std::io::Error> { - log::info!( - target: LOG_TARGET, - "Writing weight consumption for Para {}-{} for block: #{}", - para.relay_chain, para.para_id, consumption.block_number - ); - - let file_path = file_path(para); - let file = OpenOptions::new().write(true).create(true).append(true).open(file_path)?; - - let mut wtr = WriterBuilder::new().from_writer(file); - - // The data is stored in the sequence described at the beginning of the file. - wtr.write_record(&[ - // Block number: - consumption.block_number.to_string(), - // Timestamp: - consumption.timestamp.to_string(), - // Reftime consumption: - consumption.ref_time.normal.to_string(), - consumption.ref_time.operational.to_string(), - consumption.ref_time.mandatory.to_string(), - // Proof size: - consumption.proof_size.normal.to_string(), - consumption.proof_size.operational.to_string(), - consumption.proof_size.mandatory.to_string(), - ])?; - - wtr.flush() -} - async fn weight_consumption( api: OnlineClient, block_number: u32, diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..613bfe5 --- /dev/null +++ b/config.toml @@ -0,0 +1,2 @@ +output_directory = "out/" +parachains_file = "parachains.json" diff --git a/routes/Cargo.toml b/routes/Cargo.toml index 28d51fa..6950623 100644 --- a/routes/Cargo.toml +++ b/routes/Cargo.toml @@ -13,4 +13,7 @@ rocket_cors = "0.6.0" serde_json = "1.0.108" types = { path = "../types" } -shared = { path = "../shared" } +shared = { path = "../shared", features = ["test-utils"]} + +[dev-dependencies] +maplit = "1.0.2" diff --git a/routes/config.toml b/routes/config.toml new file mode 100644 index 0000000..2e2d9d6 --- /dev/null +++ b/routes/config.toml @@ -0,0 +1,2 @@ +output_directory = "mock-out/" +parachains_file = "mock-parachains.json" diff --git a/routes/mock-out/Polkadot-1000.csv b/routes/mock-out/Polkadot-1000.csv new file mode 100644 index 0000000..e69de29 diff --git a/routes/mock-parachains.json b/routes/mock-parachains.json new file mode 100644 index 0000000..7585d2a --- /dev/null +++ b/routes/mock-parachains.json @@ -0,0 +1,8 @@ +[ + { + "name": "Acala", + "rpc_url": "wss://acala-rpc.dwellir.com", + "para_id": 2000, + "relay_chain": "Polkadot" + } +] \ No newline at end of file diff --git a/routes/src/consumption.rs b/routes/src/consumption.rs index df888c6..ca9cd70 100644 --- a/routes/src/consumption.rs +++ b/routes/src/consumption.rs @@ -16,35 +16,44 @@ use crate::Error; use csv::ReaderBuilder; use rocket::get; -use shared::{file_path, parachain}; +use shared::{output_file_path, parachain}; use std::fs::File; use types::{ParaId, Timestamp, WeightConsumption}; /// Query the consumption data of a parachain. /// /// This will return an error in case there is no data associated with the specific parachain. -#[get("/consumption//?<_start>&<_end>&&")] +#[get("/consumption//?&&&")] pub fn consumption( relay: &str, para_id: ParaId, - _start: Option, - _end: Option, + start: Option, + end: Option, page: Option, page_size: Option, ) -> Result { let para = parachain(relay.into(), para_id).ok_or(Error::NotRegistered)?; - let file = File::open(file_path(para)).map_err(|_| Error::ConsumptionDataNotFound)?; + let file = File::open(output_file_path(para)).map_err(|_| Error::ConsumptionDataNotFound)?; let mut rdr = ReaderBuilder::new().has_headers(false).from_reader(file); let (page, page_size) = (page.unwrap_or_default(), page_size.unwrap_or(u32::MAX)); + let (start, end) = (start.unwrap_or_default(), end.unwrap_or(Timestamp::MAX)); let weight_consumptions: Vec = rdr .deserialize::() - .filter_map(|result| result.ok()) - .skip((page.saturating_add(page_size)) as usize) + .filter_map(|result| match result { + Ok(consumption) if consumption.timestamp >= start && consumption.timestamp <= end => + Some(consumption), + _ => None, + }) + .collect(); + + let paginated_weight_consumptions: Vec = weight_consumptions + .into_iter() + .skip(page.saturating_mul(page_size) as usize) .take(page_size as usize) .collect(); - serde_json::to_string(&weight_consumptions).map_err(|_| Error::InvalidData) + serde_json::to_string(&paginated_weight_consumptions).map_err(|_| Error::InvalidData) } diff --git a/routes/src/register.rs b/routes/src/register.rs index d38e8cb..a8b7826 100644 --- a/routes/src/register.rs +++ b/routes/src/register.rs @@ -15,7 +15,7 @@ use crate::*; use rocket::{post, serde::json::Json}; -use shared::{parachain, PARACHAINS}; +use shared::{parachain, parachains_file_path}; use std::{ fs::{File, OpenOptions}, io::{Read, Seek, Write}, @@ -27,9 +27,8 @@ use types::Parachain; pub fn register_para(para: Json) -> Result<(), Error> { let mut file = OpenOptions::new() .read(true) - .write(true) .create(true) - .open(PARACHAINS) + .open(parachains_file_path()) .map_err(|_| Error::ParasDataNotFound)?; let mut content = String::new(); diff --git a/routes/tests/consumption.rs b/routes/tests/consumption.rs new file mode 100644 index 0000000..9c13d68 --- /dev/null +++ b/routes/tests/consumption.rs @@ -0,0 +1,34 @@ +// This file is part of RegionX. +// +// RegionX is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// RegionX is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with RegionX. If not, see . + +#[cfg(test)] +use rocket::local::blocking::Client; +use rocket::{http::Status, routes}; +use routes::consumption::consumption; + +mod mock; +use mock::MockEnvironment; + +#[test] +fn getting_all_consumption_data_works() { + MockEnvironment::new().execute_with(|| { + let rocket = rocket::build().mount("/", routes![consumption]); + let client = Client::tracked(rocket).expect("valid rocket instance"); + let response = client.get("/consumption/polkadot/2000").dispatch(); + assert_eq!(response.status(), Status::Ok); + }); +} + +// TODO: https://github.com/RegionX-Labs/CorespaceWeigher/issues/11 diff --git a/routes/tests/mock.rs b/routes/tests/mock.rs new file mode 100644 index 0000000..22b3dc1 --- /dev/null +++ b/routes/tests/mock.rs @@ -0,0 +1,90 @@ +// This file is part of RegionX. +// +// RegionX is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// RegionX is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with RegionX. If not, see . + +#[cfg(test)] +use maplit::hashmap; +use shared::{delete_conspumption, write_consumption}; +use std::collections::HashMap; +use types::{ParaId, Parachain, RelayChain, RelayChain::*, WeightConsumption}; + +#[derive(Default)] +pub struct MockEnvironment { + pub weight_consumptions: HashMap>, +} + +impl MockEnvironment { + pub fn new() -> Self { + // Initialize some mock data: + let mock = MockEnvironment { + weight_consumptions: hashmap! { + mock_para(Polkadot, 2000) => vec![ + WeightConsumption { + block_number: 1, + timestamp: 0, + ref_time: (0.5, 0.3, 0.2).into(), + proof_size: (0.5, 0.3, 0.2).into(), + }, + WeightConsumption { + block_number: 2, + timestamp: 6, + ref_time: (0.1, 0.4, 0.2).into(), + proof_size: (0.2, 0.3, 0.3).into(), + }, + WeightConsumption { + block_number: 3, + timestamp: 12, + ref_time: (0.0, 0.2, 0.4).into(), + proof_size: (0.1, 0.0, 0.3).into(), + }, + ], + mock_para(Polkadot, 2005) => vec![ + WeightConsumption { + block_number: 1, + timestamp: 0, + ref_time: (0.8, 0.0, 0.1).into(), + proof_size: (0.6, 0.2, 0.1).into(), + }, + ], + }, + }; + + for (para, weight_consumptions) in &mock.weight_consumptions { + weight_consumptions.iter().for_each(|consumption| { + write_consumption(para.clone(), consumption.clone()) + .expect("Failed to write conusumption data"); + }); + } + + mock + } + + pub fn execute_with(&self, execute: impl FnOnce() -> R) -> R { + let result = execute(); + // Cleanup the mock data after the test is complete: + for para in self.weight_consumptions.keys() { + delete_conspumption(para.clone()); + } + result + } +} + +fn mock_para(relay: RelayChain, para_id: ParaId) -> Parachain { + Parachain { + name: format!("{}-{}", relay, para_id), + rpc_url: format!("wss://{}-{}.com", relay, para_id), + para_id, + relay_chain: relay, + } +} diff --git a/routes/tests/register.rs b/routes/tests/register.rs new file mode 100644 index 0000000..4a67bc3 --- /dev/null +++ b/routes/tests/register.rs @@ -0,0 +1,16 @@ +// This file is part of RegionX. +// +// RegionX is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// RegionX is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with RegionX. If not, see . + +// TODO: https://github.com/RegionX-Labs/CorespaceWeigher/issues/11 diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..e9d4cf2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly" +components = [ "rustfmt", "clippy" ] diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 3b3e2af..2db084f 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -9,5 +9,13 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -types = { path = "../types" } +csv = "1.3.0" +log = "0.4" +toml = "0.5.8" +serde = "1.0.193" serde_json = "1.0.108" + +types = { path = "../types" } + +[features] +test-utils = [] diff --git a/shared/src/lib.rs b/shared/src/lib.rs index bde3836..529a8ef 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -13,13 +13,25 @@ // You should have received a copy of the GNU General Public License // along with RegionX. If not, see . -use std::{fs::File, io::Read}; -use types::{ParaId, Parachain, RelayChain}; +use csv::WriterBuilder; +use std::{ + fs::{File, OpenOptions}, + io::Read, +}; +use types::{ParaId, Parachain, RelayChain, WeightConsumption}; -pub const PARACHAINS: &str = "parachains.json"; +const LOG_TARGET: &str = "tracker"; + +pub const CONFIG_FILE: &str = "config.toml"; + +#[derive(serde::Deserialize)] +struct Config { + output_directory: String, + parachains_file: String, +} pub fn parachains() -> Vec { - let mut file = File::open(PARACHAINS).expect("Hardcoded path is known good; qed"); + let mut file = File::open(parachains_file_path()).expect("Hardcoded path is known good; qed"); let mut content = String::new(); if file.read_to_string(&mut content).is_ok() { @@ -30,7 +42,6 @@ pub fn parachains() -> Vec { } } -#[allow(dead_code)] pub fn parachain(relay_chain: RelayChain, para_id: ParaId) -> Option { parachains() .iter() @@ -38,12 +49,65 @@ pub fn parachain(relay_chain: RelayChain, para_id: ParaId) -> Option .cloned() } -pub fn file_path(para: Parachain) -> String { - format!("out/{}-{}.csv", para.relay_chain, para.para_id) +pub fn parachains_file_path() -> String { + let config_str = std::fs::read_to_string("config.toml").expect("Failed to read config file"); + + let config: Config = toml::from_str(&config_str).expect("Failed to parse config file"); + + config.parachains_file +} + +pub fn output_file_path(para: Parachain) -> String { + let config_str = std::fs::read_to_string("config.toml").expect("Failed to read config file"); + + let config: Config = toml::from_str(&config_str).expect("Failed to parse config file"); + + format!("{}/{}-{}.csv", config.output_directory, para.relay_chain, para.para_id) } -#[allow(dead_code)] pub fn round_to(number: f32, decimals: i32) -> f32 { let factor = 10f32.powi(decimals); (number * factor).round() / factor } + +pub fn write_consumption( + para: Parachain, + consumption: WeightConsumption, +) -> Result<(), std::io::Error> { + log::info!( + target: LOG_TARGET, + "Writing weight consumption for Para {}-{} for block: #{}", + para.relay_chain, para.para_id, consumption.block_number + ); + + let output_file_path = output_file_path(para); + let file = OpenOptions::new().create(true).append(true).open(output_file_path)?; + + let mut wtr = WriterBuilder::new().from_writer(file); + + // The data is stored in the sequence described at the beginning of the file. + wtr.write_record(&[ + // Block number: + consumption.block_number.to_string(), + // Timestamp: + consumption.timestamp.to_string(), + // Reftime consumption: + consumption.ref_time.normal.to_string(), + consumption.ref_time.operational.to_string(), + consumption.ref_time.mandatory.to_string(), + // Proof size: + consumption.proof_size.normal.to_string(), + consumption.proof_size.operational.to_string(), + consumption.proof_size.mandatory.to_string(), + ])?; + + wtr.flush() +} + +// There isn't a good reason to use this other than for testing. +#[cfg(feature = "test-utils")] +pub fn delete_conspumption(para: Parachain) { + let output_file_path = output_file_path(para); + + std::fs::remove_file(output_file_path).expect("Failed to delete consumption"); +} diff --git a/types/src/lib.rs b/types/src/lib.rs index bc53312..7483765 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -23,7 +23,7 @@ pub type Timestamp = u64; /// Type used for identifying parachains. pub type ParaId = u32; -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize, Hash)] #[serde(crate = "rocket::serde")] pub enum RelayChain { Polkadot, @@ -49,7 +49,7 @@ impl From<&str> for RelayChain { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)] #[serde(crate = "rocket::serde")] pub struct Parachain { /// Name of the parachain. @@ -64,7 +64,7 @@ pub struct Parachain { pub relay_chain: RelayChain, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct WeightConsumption { /// The block number for which the weight consumption is related to. pub block_number: u32, @@ -76,7 +76,7 @@ pub struct WeightConsumption { pub proof_size: DispatchClassConsumption, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct DispatchClassConsumption { /// The percentage of the weight used by user submitted extrinsics compared to the /// maximum potential.