This commits adds a migration binary to moves clients on a postgresql…

… db to a smart contract db

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "rita_exit",
     "rita_client",
     "rita_client_registration",
+    "rita_db_migration",
     "rita_bin",
     "test_runner",
     "integration_tests",

rita_bin/Cargo.toml

@@ -13,44 +13,54 @@ path = "src/exit.rs"
 name = "rita"
 path = "src/client.rs"
 
+[[bin]]
+name = "rita_db_migration"
+path = "src/database_migration.rs"
+
 [[bin]]
 name = "rita_extender"
 path = "src/extender.rs"
 
 [dependencies]
 althea_kernel_interface = { path = "../althea_kernel_interface" }
-althea_types = { path = "../althea_types"}
+althea_types = { path = "../althea_types" }
 clu = { path = "../clu" }
-actix-async = {package="actix", version = "0.13"}
-actix-web-async = {package="actix-web", version = "4.3", default_features = false, features= ["openssl"] }
+actix-async = { package = "actix", version = "0.13" }
+actix-web-async = { package = "actix-web", version = "4.3", default_features = false, features = [
+    "openssl",
+] }
 docopt = "1.1"
 compressed_log = "0.5"
 settings = { path = "../settings" }
 diesel = { version = "1.4", features = ["postgres", "r2d2"] }
 log = { version = "0.4", features = ["release_max_level_info"] }
 serde = "1.0"
 serde_json = "1.0"
-arrayvec = {version= "0.7", features = ["serde"]}
+arrayvec = { version = "0.7", features = ["serde"] }
 hex-literal = "0.3"
 openssl-probe = "0.1"
 env_logger = "0.10"
 lazy_static = "1.4"
 phonenumber = "0.3"
 r2d2 = "0.8"
-ctrlc = {version = "3.2.1", features = ["termination"]}
+ctrlc = { version = "3.2.1", features = ["termination"] }
 rita_common = { path = "../rita_common", default-features = false }
-rita_client = { path = "../rita_client",  default-features = false }
+rita_client = { path = "../rita_client", default-features = false }
+rita_client_registration = { path = "../rita_client_registration", default-features = false }
+rita_db_migration = { path = "../rita_db_migration", default-features = false }
 rita_exit = { path = "../rita_exit", default-features = false }
-rita_extender = { path = "../rita_extender", default-features =  false}
-flate2 = { version = "1.0", features = ["rust_backend"], default-features = false }
+rita_extender = { path = "../rita_extender", default-features = false }
+flate2 = { version = "1.0", features = [
+    "rust_backend",
+], default-features = false }
 reqwest = { version = "0.11", features = ["blocking", "json"] }
-jemallocator = {version = "0.5", optional = true}
+jemallocator = { version = "0.5", optional = true }
 # we don't call or us OpenSSL directly in this codebase, but by adding
 # this dependency with this feature we can enforce that openssl is compiled
 # in 'vendored' mode all the way down the tree. What this means is that we use
 # an openssl implementation from the crate and not from the system, simplifying
 # our build process for a lot of cross-compile situations
-openssl = {version = "0.10", features = ["vendored"]}
+openssl = { version = "0.10", features = ["vendored"] }
 ipnetwork = "0.20"
 actix-rt = "2"
 
@@ -63,7 +73,15 @@ dash_debug = []
 # changes operator urls
 operator_debug = []
 dev_env = []
-legacy_integration_test = ["development", "althea_kernel_interface/integration_test", "rita_common/legacy_integration_test"]
-development = ["rita_common/dash_debug","rita_client/operator_debug", "rita_exit/operator_debug"]
+legacy_integration_test = [
+    "development",
+    "althea_kernel_interface/integration_test",
+    "rita_common/legacy_integration_test",
+]
+development = [
+    "rita_common/dash_debug",
+    "rita_client/operator_debug",
+    "rita_exit/operator_debug",
+]
 # Op tools dev environement
 optools_dev_env = ["rita_client/dev_env", "rita_exit/dev_env"]

rita_bin/src/database_migration.rs

@@ -0,0 +1,64 @@
+//! This binary is run to have access to a postgresql db, takes its client info and writes it into
+//! smart contract. This is necessary setup to move existing registered clients from the previous
+//! sql db format to a smart contract
+
+#![warn(clippy::all)]
+#![allow(clippy::pedantic)]
+#![forbid(unsafe_code)]
+
+use docopt::Docopt;
+use log::{error, info};
+use rita_client_registration::register_client_batch_loop::register_client_batch_loop;
+use rita_db_migration::start_db_migration;
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+pub struct Args {
+    pub sql_db_url: String,
+    pub address: String,
+    pub web3_url: String,
+    pub private_key: String,
+}
+
+fn main() {
+    let args: Args = Docopt::new(get_arg_usage())
+        .and_then(|d| d.deserialize())
+        .unwrap_or_else(|e| e.exit());
+
+    let db_url = args.sql_db_url;
+    let contract_addr = args
+        .address
+        .parse()
+        .expect("Please provide a valid eth contract addr");
+    let web3_url = args.web3_url;
+    let private_key = args
+        .private_key
+        .parse()
+        .expect("Please provide a valid eth private key with funds");
+
+    let system = actix_async::System::new();
+
+    // Start registration loop
+    register_client_batch_loop(web3_url, contract_addr, private_key);
+
+    match start_db_migration(db_url) {
+        Ok(_) => println!("Successfully migrated all clients!"),
+        Err(e) => println!("Failed to migrate clients with {}", e),
+    }
+
+    if let Err(e) = system.run() {
+        error!("Starting Rita DB migration failed with {}", e);
+    }
+    info!("Started Rita DB migration!");
+}
+
+pub fn get_arg_usage() -> String {
+    "Usage: rita_db_migration [--db_url=<db_url>] [--address=<address>] [--web3_url=<web3_url>] [--private_key=<private_key>]
+Options:
+    -u, --db_url=<db_url>           Postgresql db url
+    -a, --address=<address>         Smart Contract address
+    -w, --web3_url=<web3_url>       Web3 url
+    -p, --private_key=<private_key> Our Private key
+About: 
+    Db migration binary".to_string()
+}

rita_client_registration/src/lib.rs

@@ -70,7 +70,7 @@ fn get_texts_sent(key: WgKey) -> u8 {
     *TEXTS_SENT.read().unwrap().get(&key).unwrap_or(&0u8)
 }
 
-fn add_client_to_reg_batch(id: Identity) {
+pub fn add_client_to_reg_batch(id: Identity) {
     TX_BATCH.write().unwrap().insert(id);
 }
 

rita_db_migration/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "rita_db_migration"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+r2d2 = "0.8"
+diesel = { version = "1.4", features = ["postgres", "r2d2"] }
+log = { version = "0.4", features = ["release_max_level_info"] }
+dotenv = "0.15"
+althea_types = { path = "../althea_types" }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+rita_client_registration = { path = "../rita_client_registration" }

rita_db_migration/src/error.rs

@@ -0,0 +1,19 @@
+use std::{
+    error::Error,
+    fmt::{Display, Formatter, Result as FmtResult},
+};
+
+#[derive(Debug)]
+pub enum RitaDBMigrationError {
+    MiscStringError(String),
+}
+
+impl Display for RitaDBMigrationError {
+    fn fmt(&self, f: &mut Formatter) -> FmtResult {
+        match self {
+            RitaDBMigrationError::MiscStringError(a) => write!(f, "{a}",),
+        }
+    }
+}
+
+impl Error for RitaDBMigrationError {}

rita_db_migration/src/lib.rs

@@ -0,0 +1,96 @@
+#[macro_use]
+extern crate log;
+#[macro_use]
+extern crate diesel;
+#[macro_use]
+extern crate serde_derive;
+
+pub mod error;
+pub mod models;
+pub mod schema;
+
+use crate::schema::clients::dsl::clients;
+use althea_types::Identity;
+use diesel::{r2d2::ConnectionManager, PgConnection, RunQueryDsl};
+use error::RitaDBMigrationError;
+use models::Client;
+use r2d2::PooledConnection;
+use rita_client_registration::add_client_to_reg_batch;
+
+pub fn start_db_migration(db_url: String) -> Result<(), RitaDBMigrationError> {
+    // Validate that db_url and contract_addr are valid
+    if !(db_url.contains("postgres://")
+        || db_url.contains("postgresql://")
+        || db_url.contains("psql://"))
+    {
+        panic!("You must provide a valid postgressql database uri!");
+    }
+
+    let db_conn = get_database_connection(db_url)?;
+
+    if let Ok(clients_list) = clients.load::<models::Client>(&db_conn) {
+        info!(
+            "Recieved a valid client list with {} entries",
+            clients_list.len()
+        );
+
+        add_clients_to_reg_queue(clients_list)
+    } else {
+        return Err(RitaDBMigrationError::MiscStringError(
+            "Unable to get db clients".to_string(),
+        ));
+    }
+
+    Ok(())
+}
+
+fn add_clients_to_reg_queue(client_list: Vec<Client>) {
+    for c in client_list {
+        let id = Identity {
+            mesh_ip: match c.mesh_ip.parse() {
+                Ok(a) => a,
+                Err(e) => {
+                    error!("Cannot parse client {:?} mesh ip! with {}", c, e);
+                    continue;
+                }
+            },
+            eth_address: match c.eth_address.parse() {
+                Ok(a) => a,
+                Err(e) => {
+                    error!("Cannot parse client {:?} eth addr! with {}", c, e);
+                    continue;
+                }
+            },
+            wg_public_key: match c.wg_pubkey.parse() {
+                Ok(a) => a,
+                Err(e) => {
+                    error!("Cannot parse client {:?} wg key! with {}", c, e);
+                    continue;
+                }
+            },
+            nickname: None,
+        };
+
+        add_client_to_reg_batch(id);
+    }
+}
+
+pub fn get_database_connection(
+    db_url: String,
+) -> Result<PooledConnection<ConnectionManager<PgConnection>>, RitaDBMigrationError> {
+    let manager = ConnectionManager::new(db_url);
+    let pool = r2d2::Pool::builder()
+        .max_size(1)
+        .build(manager)
+        .expect("Failed to create pool.");
+
+    match pool.try_get() {
+        Some(connection) => Ok(connection),
+        None => {
+            error!("No available db connection!");
+            Err(RitaDBMigrationError::MiscStringError(
+                "No Database connection available!".to_string(),
+            ))
+        }
+    }
+}

rita_db_migration/src/models.rs

@@ -0,0 +1,37 @@
+#![allow(clippy::extra_unused_lifetimes)]
+use crate::schema::{assigned_ips, clients};
+
+#[derive(Queryable, Serialize, Deserialize, Debug, Insertable, Clone, Default)]
+#[table_name = "clients"]
+pub struct Client {
+    pub mesh_ip: String,
+    pub wg_pubkey: String,
+    pub wg_port: i32,
+    pub eth_address: String,
+    pub internal_ip: String,
+    pub internet_ipv6: String,
+    pub nickname: String,
+    pub email: String,
+    pub phone: String,
+    pub country: String,
+    pub email_code: String,
+    pub verified: bool,
+    pub email_sent_time: i64,
+    pub text_sent: i32,
+    pub last_seen: i64,
+    pub last_balance_warning_time: i64,
+}
+
+/// This struct holds information about the ipv6 subnets being assigned to clients who connect.
+/// The vector available subnets is a stack that has a list of available subnets to use. This stack gets populated whenever
+/// a client gets removed from the database. It is stored as a string of indecies, for example, "1,24,36"
+/// The iterative index stores the index at which we assign a subnet to a client
+/// For example, if our exit subnet is fd00::1000/120 and our client subnets are /124, index 0 represents
+/// fd00::1000/124 index 1 represents fd00::1010/124, 2 is fd00::1120/124 etc...
+#[derive(Queryable, Serialize, Deserialize, Debug, Insertable, Clone, Default)]
+#[table_name = "assigned_ips"]
+pub struct AssignedIps {
+    pub subnet: String,
+    pub available_subnets: String,
+    pub iterative_index: i64,
+}