Query based on timestamp (#10)

* Query based on timestamp

* toolchain

* make clippy happy

* init tests

* fix

* mock environment setup

* add todos

bin/tracker/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-csv = "1.3.0"
 log = "0.4"
 env_logger = "0.10.1"
 subxt = "0.32.1"

bin/tracker/src/main.rs

@@ -43,9 +43,7 @@
 
 const LOG_TARGET: &str = "tracker";
 
-use csv::WriterBuilder;
-use shared::{file_path, parachains, round_to};
-use std::fs::OpenOptions;
+use shared::{parachains, round_to, write_consumption};
 use subxt::{blocks::Block, utils::H256, OnlineClient, PolkadotConfig};
 use types::{Parachain, Timestamp, WeightConsumption};
 
@@ -115,40 +113,6 @@ async fn note_new_block(
 	Ok(())
 }
 
-fn write_consumption(
-	para: Parachain,
-	consumption: WeightConsumption,
-) -> Result<(), std::io::Error> {
-	log::info!(
-		target: LOG_TARGET,
-		"Writing weight consumption for Para {}-{} for block: #{}",
-		para.relay_chain, para.para_id, consumption.block_number
-	);
-
-	let file_path = file_path(para);
-	let file = OpenOptions::new().write(true).create(true).append(true).open(file_path)?;
-
-	let mut wtr = WriterBuilder::new().from_writer(file);
-
-	// The data is stored in the sequence described at the beginning of the file.
-	wtr.write_record(&[
-		// Block number:
-		consumption.block_number.to_string(),
-		// Timestamp:
-		consumption.timestamp.to_string(),
-		// Reftime consumption:
-		consumption.ref_time.normal.to_string(),
-		consumption.ref_time.operational.to_string(),
-		consumption.ref_time.mandatory.to_string(),
-		// Proof size:
-		consumption.proof_size.normal.to_string(),
-		consumption.proof_size.operational.to_string(),
-		consumption.proof_size.mandatory.to_string(),
-	])?;
-
-	wtr.flush()
-}
-
 async fn weight_consumption(
 	api: OnlineClient<PolkadotConfig>,
 	block_number: u32,

config.toml

@@ -0,0 +1,2 @@
+output_directory = "out/"
+parachains_file = "parachains.json"

routes/Cargo.toml

@@ -13,4 +13,7 @@ rocket_cors = "0.6.0"
 serde_json = "1.0.108"
 
 types = { path = "../types" }
-shared = { path = "../shared" }
+shared = { path = "../shared", features = ["test-utils"]}
+
+[dev-dependencies]
+maplit = "1.0.2"

routes/config.toml

@@ -0,0 +1,2 @@
+output_directory = "mock-out/"
+parachains_file = "mock-parachains.json"

routes/mock-parachains.json

@@ -0,0 +1,8 @@
+[
+  {
+    "name": "Acala",
+    "rpc_url": "wss://acala-rpc.dwellir.com",
+    "para_id": 2000,
+    "relay_chain": "Polkadot"
+  }
+]

routes/src/consumption.rs

@@ -16,35 +16,44 @@
 use crate::Error;
 use csv::ReaderBuilder;
 use rocket::get;
-use shared::{file_path, parachain};
+use shared::{output_file_path, parachain};
 use std::fs::File;
 use types::{ParaId, Timestamp, WeightConsumption};
 
 /// Query the consumption data of a parachain.
 ///
 /// This will return an error in case there is no data associated with the specific parachain.
-#[get("/consumption/<relay>/<para_id>?<_start>&<_end>&<page>&<page_size>")]
+#[get("/consumption/<relay>/<para_id>?<start>&<end>&<page>&<page_size>")]
 pub fn consumption(
 	relay: &str,
 	para_id: ParaId,
-	_start: Option<Timestamp>,
-	_end: Option<Timestamp>,
+	start: Option<Timestamp>,
+	end: Option<Timestamp>,
 	page: Option<u32>,
 	page_size: Option<u32>,
 ) -> Result<String, Error> {
 	let para = parachain(relay.into(), para_id).ok_or(Error::NotRegistered)?;
 
-	let file = File::open(file_path(para)).map_err(|_| Error::ConsumptionDataNotFound)?;
+	let file = File::open(output_file_path(para)).map_err(|_| Error::ConsumptionDataNotFound)?;
 	let mut rdr = ReaderBuilder::new().has_headers(false).from_reader(file);
 
 	let (page, page_size) = (page.unwrap_or_default(), page_size.unwrap_or(u32::MAX));
+	let (start, end) = (start.unwrap_or_default(), end.unwrap_or(Timestamp::MAX));
 
 	let weight_consumptions: Vec<WeightConsumption> = rdr
 		.deserialize::<WeightConsumption>()
-		.filter_map(|result| result.ok())
-		.skip((page.saturating_add(page_size)) as usize)
+		.filter_map(|result| match result {
+			Ok(consumption) if consumption.timestamp >= start && consumption.timestamp <= end =>
+				Some(consumption),
+			_ => None,
+		})
+		.collect();
+
+	let paginated_weight_consumptions: Vec<WeightConsumption> = weight_consumptions
+		.into_iter()
+		.skip(page.saturating_mul(page_size) as usize)
 		.take(page_size as usize)
 		.collect();
 
-	serde_json::to_string(&weight_consumptions).map_err(|_| Error::InvalidData)
+	serde_json::to_string(&paginated_weight_consumptions).map_err(|_| Error::InvalidData)
 }

routes/src/register.rs

@@ -15,7 +15,7 @@
 
 use crate::*;
 use rocket::{post, serde::json::Json};
-use shared::{parachain, PARACHAINS};
+use shared::{parachain, parachains_file_path};
 use std::{
 	fs::{File, OpenOptions},
 	io::{Read, Seek, Write},
@@ -27,9 +27,8 @@ use types::Parachain;
 pub fn register_para(para: Json<Parachain>) -> Result<(), Error> {
 	let mut file = OpenOptions::new()
 		.read(true)
-		.write(true)
 		.create(true)
-		.open(PARACHAINS)
+		.open(parachains_file_path())
 		.map_err(|_| Error::ParasDataNotFound)?;
 
 	let mut content = String::new();

routes/tests/consumption.rs

@@ -0,0 +1,34 @@
+// This file is part of RegionX.
+//
+// RegionX is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// RegionX is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with RegionX.  If not, see <https://www.gnu.org/licenses/>.
+
+#[cfg(test)]
+use rocket::local::blocking::Client;
+use rocket::{http::Status, routes};
+use routes::consumption::consumption;
+
+mod mock;
+use mock::MockEnvironment;
+
+#[test]
+fn getting_all_consumption_data_works() {
+	MockEnvironment::new().execute_with(|| {
+		let rocket = rocket::build().mount("/", routes![consumption]);
+		let client = Client::tracked(rocket).expect("valid rocket instance");
+		let response = client.get("/consumption/polkadot/2000").dispatch();
+		assert_eq!(response.status(), Status::Ok);
+	});
+}
+
+// TODO: https://github.com/RegionX-Labs/CorespaceWeigher/issues/11

routes/tests/mock.rs

@@ -0,0 +1,90 @@
+// This file is part of RegionX.
+//
+// RegionX is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// RegionX is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with RegionX.  If not, see <https://www.gnu.org/licenses/>.
+
+#[cfg(test)]
+use maplit::hashmap;
+use shared::{delete_conspumption, write_consumption};
+use std::collections::HashMap;
+use types::{ParaId, Parachain, RelayChain, RelayChain::*, WeightConsumption};
+
+#[derive(Default)]
+pub struct MockEnvironment {
+	pub weight_consumptions: HashMap<Parachain, Vec<WeightConsumption>>,
+}
+
+impl MockEnvironment {
+	pub fn new() -> Self {
+		// Initialize some mock data:
+		let mock = MockEnvironment {
+			weight_consumptions: hashmap! {
+				mock_para(Polkadot, 2000) => vec![
+					WeightConsumption {
+						block_number: 1,
+						timestamp: 0,
+						ref_time: (0.5, 0.3, 0.2).into(),
+						proof_size: (0.5, 0.3, 0.2).into(),
+					},
+					WeightConsumption {
+						block_number: 2,
+						timestamp: 6,
+						ref_time: (0.1, 0.4, 0.2).into(),
+						proof_size: (0.2, 0.3, 0.3).into(),
+					},
+					WeightConsumption {
+						block_number: 3,
+						timestamp: 12,
+						ref_time: (0.0, 0.2, 0.4).into(),
+						proof_size: (0.1, 0.0, 0.3).into(),
+					},
+				],
+				mock_para(Polkadot, 2005) => vec![
+					WeightConsumption {
+						block_number: 1,
+						timestamp: 0,
+						ref_time: (0.8, 0.0, 0.1).into(),
+						proof_size: (0.6, 0.2, 0.1).into(),
+					},
+				],
+			},
+		};
+
+		for (para, weight_consumptions) in &mock.weight_consumptions {
+			weight_consumptions.iter().for_each(|consumption| {
+				write_consumption(para.clone(), consumption.clone())
+					.expect("Failed to write conusumption data");
+			});
+		}
+
+		mock
+	}
+
+	pub fn execute_with<R>(&self, execute: impl FnOnce() -> R) -> R {
+		let result = execute();
+		// Cleanup the mock data after the test is complete:
+		for para in self.weight_consumptions.keys() {
+			delete_conspumption(para.clone());
+		}
+		result
+	}
+}
+
+fn mock_para(relay: RelayChain, para_id: ParaId) -> Parachain {
+	Parachain {
+		name: format!("{}-{}", relay, para_id),
+		rpc_url: format!("wss://{}-{}.com", relay, para_id),
+		para_id,
+		relay_chain: relay,
+	}
+}

routes/tests/register.rs

@@ -0,0 +1,16 @@
+// This file is part of RegionX.
+//
+// RegionX is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// RegionX is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with RegionX.  If not, see <https://www.gnu.org/licenses/>.
+
+// TODO: https://github.com/RegionX-Labs/CorespaceWeigher/issues/11

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "nightly"
+components = [ "rustfmt", "clippy" ]

shared/Cargo.toml

@@ -9,5 +9,13 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-types = { path = "../types" }
+csv = "1.3.0"
+log = "0.4"
+toml = "0.5.8"
+serde = "1.0.193"
 serde_json = "1.0.108"
+
+types = { path = "../types" }
+
+[features]
+test-utils = []