From 5c8cff900ccd591f635c7f682f2f30f30d97e5fa Mon Sep 17 00:00:00 2001 From: Justin Kilpatrick Date: Wed, 4 Dec 2024 19:20:14 -0500 Subject: [PATCH] Remove db migration test The diesel version is now so old it's throwing Rust warnings, I'll pull this out of the history when/if I need to use it. This finaly lets us remove all the postgres related stuff --- .github/workflows/rust.yml | 12 +- Cargo.lock | 93 ------- Cargo.toml | 1 - README.md | 8 +- integration_tests/Cargo.toml | 3 - integration_tests/container/Dockerfile | 6 +- integration_tests/src/db_migration_test.rs | 176 ------------- integration_tests/src/lib.rs | 1 - integration_tests/src/setup_utils/database.rs | 130 --------- integration_tests/src/setup_utils/mod.rs | 1 - integration_tests/src/utils.rs | 2 +- rita_bin/Cargo.toml | 6 - rita_bin/src/contract-util.rs | 249 ------------------ rita_db_migration/Cargo.toml | 20 -- rita_db_migration/src/error.rs | 19 -- rita_db_migration/src/lib.rs | 145 ---------- rita_db_migration/src/models.rs | 37 --- rita_db_migration/src/schema.rs | 28 -- scripts/integration_tests/all-up-test-ci.sh | 5 - test_runner/src/main.rs | 3 - 20 files changed, 8 insertions(+), 937 deletions(-) delete mode 100644 integration_tests/src/db_migration_test.rs delete mode 100644 integration_tests/src/setup_utils/database.rs delete mode 100644 rita_bin/src/contract-util.rs delete mode 100644 rita_db_migration/Cargo.toml delete mode 100644 rita_db_migration/src/error.rs delete mode 100644 rita_db_migration/src/lib.rs delete mode 100644 rita_db_migration/src/models.rs delete mode 100644 rita_db_migration/src/schema.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 093e89cc1..e5bbbb6d2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -123,14 +123,4 @@ jobs: with: cache-on-failure: true - name: Run integration test - run: bash scripts/integration_tests/all-up-test-ci.sh CONTRACT_TEST - integration-test-db-migration: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - - name: Run integration test - run: bash scripts/integration_tests/all-up-test-ci.sh MIGRATION_TEST + run: bash scripts/integration_tests/all-up-test-ci.sh CONTRACT_TEST \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 6d41e98d1..626203e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,40 +1207,6 @@ dependencies = [ "syn 2.0.79", ] -[[package]] -name = "diesel" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "diesel_derives", - "pq-sys", - "r2d2", -] - -[[package]] -name = "diesel_derives" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "diesel_migrations" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" -dependencies = [ - "migrations_internals", - "migrations_macros", -] - [[package]] name = "digest" version = "0.10.7" @@ -1264,12 +1230,6 @@ dependencies = [ "strsim 0.10.0", ] -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - [[package]] name = "ed25519" version = "1.5.3" @@ -1917,8 +1877,6 @@ dependencies = [ "crossbeam", "ctrlc", "deep_space", - "diesel", - "diesel_migrations", "docopt", "env_logger", "exit_trust_root", @@ -1933,7 +1891,6 @@ dependencies = [ "phonenumber", "rita_client", "rita_common", - "rita_db_migration", "rita_exit", "settings", "tokio", @@ -2192,27 +2149,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "migrations_internals" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" -dependencies = [ - "diesel", -] - -[[package]] -name = "migrations_macros" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "mime" version = "0.3.17" @@ -2693,15 +2629,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pq-sys" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0052426df997c0cbd30789eb44ca097e3541717a7b8fa36b1c464ee7edebd" -dependencies = [ - "vcpkg", -] - [[package]] name = "proc-macro-crate" version = "3.2.0" @@ -2975,7 +2902,6 @@ dependencies = [ "clu", "compressed_log", "ctrlc", - "diesel", "docopt", "env_logger", "exit_trust_root", @@ -2990,7 +2916,6 @@ dependencies = [ "r2d2", "rita_client", "rita_common", - "rita_db_migration", "rita_exit", "rita_extender", "serde", @@ -3085,24 +3010,6 @@ dependencies = [ "web30", ] -[[package]] -name = "rita_db_migration" -version = "0.1.0" -dependencies = [ - "althea_types", - "clarity", - "crossbeam", - "diesel", - "dotenv", - "exit_trust_root", - "log", - "r2d2", - "serde", - "serde_derive", - "serde_json", - "web30", -] - [[package]] name = "rita_exit" version = "0.21.5" diff --git a/Cargo.toml b/Cargo.toml index 42ed377b4..00ebfd457 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ members = [ "rita_common", "rita_exit", "rita_client", - "rita_db_migration", "rita_bin", "test_runner", "integration_tests", diff --git a/README.md b/README.md index 96136808b..08e5c7ca2 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,19 @@ Rita supports either [Althea L1](https://github.com/althea-net/althea-L1) or any Debian: - sudo apt-get install build-essential libssl-dev libsqlite3-dev pkg-config postgresql-server-dev-all automake liboping-dev libtool perl clang + sudo apt-get install build-essential libssl-dev libsqlite3-dev pkg-config automake liboping-dev libtool perl clang Ubuntu: - sudo apt-get install build-essential libssl-dev libsqlite3-dev pkg-config postgresql-server-dev-all automake liboping-dev libtool perl clang + sudo apt-get install build-essential libssl-dev libsqlite3-dev pkg-config automake liboping-dev libtool perl clang Centos: - sudo yum install gcc gcc-c++ openssl-devel sqlite-devel make postgresql-devel automake liboping-devel libtool perl clang + sudo yum install gcc gcc-c++ openssl-devel sqlite-devel make automake liboping-devel libtool perl clang Fedora: - sudo dnf install gcc gcc-c++ openssl-devel sqlite-devel make postgresql-devel automake liboping-devel libtool perl clang + sudo dnf install gcc gcc-c++ openssl-devel sqlite-devel make automake liboping-devel libtool perl clang Arch: diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 80c6ac8d0..2202a5832 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -22,10 +22,7 @@ rita_client = { path = "../rita_client", features = ["dev_env"] } rita_common = { path = "../rita_common", features = ["integration_test"] } rita_exit = { path = "../rita_exit", features = ["dev_env"] } exit_trust_root = { path = "../exit_trust_root", features = ["development"] } -rita_db_migration = { path = "../rita_db_migration" } ctrlc = { version = "3.4.5", features = ["termination"] } -diesel = { version = "1.4", features = ["postgres", "r2d2"] } -diesel_migrations = { version = "1.4", features = ["postgres"] } awc = {workspace = true} actix-rt = {workspace = true} deep_space = {workspace = true} diff --git a/integration_tests/container/Dockerfile b/integration_tests/container/Dockerfile index 504741645..ef21ddbf2 100644 --- a/integration_tests/container/Dockerfile +++ b/integration_tests/container/Dockerfile @@ -1,11 +1,9 @@ -FROM postgres +FROM debian:bookworm ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y sudo iputils-ping iproute2 jq vim netcat-traditional default-libmysqlclient-dev libsqlite3-dev libpq-dev python3-pip bridge-utils wireguard linux-source curl git libssl-dev pkg-config build-essential ipset python3-setuptools python3-wheel dh-autoreconf procps net-tools iperf3 babeld make locales-all npm +RUN apt-get update && apt-get install -y sudo iputils-ping iproute2 jq vim netcat-traditional python3-pip bridge-utils wireguard linux-source curl git libssl-dev pkg-config build-essential ipset python3-setuptools python3-wheel dh-autoreconf procps net-tools iperf3 babeld make locales-all npm # Install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y ENV PATH=$PATH:/root/.cargo/bin/ -# Setup postgres db for Althea exits -RUN cargo install diesel_cli --force # Install Babeld for Althea routing RUN cd /var && git clone -b master https://github.com/althea-mesh/babeld.git RUN cd /var/babeld/ && make install diff --git a/integration_tests/src/db_migration_test.rs b/integration_tests/src/db_migration_test.rs deleted file mode 100644 index cd31973f9..000000000 --- a/integration_tests/src/db_migration_test.rs +++ /dev/null @@ -1,176 +0,0 @@ -use crate::{ - payments_eth::{TRASACTION_TIMEOUT, WEB3_TIMEOUT}, - setup_utils::database::start_postgres, - utils::{deploy_contracts, get_eth_node, REGISTRATION_SERVER_KEY}, -}; -use althea_types::random_identity; -use clarity::{Address, PrivateKey}; -use crossbeam::queue::SegQueue; -use diesel::{PgConnection, RunQueryDsl}; -use exit_trust_root::{ - client_db::{check_and_add_user_admin, get_all_registered_clients}, - register_client_batch_loop::register_client_batch_loop, -}; -use futures::future::{select, Either}; -use rita_db_migration::{ - get_database_connection, models::Client, schema::clients::dsl::clients, start_db_migration, -}; -use std::{ - sync::Arc, - thread, - time::{Duration, Instant}, -}; -use web30::client::Web3; - -pub const DB_URI: &str = "postgres://postgres@localhost/test"; - -/// This tests the rita_db_migration binary and veries that clients actually migrate from a postgresql db -/// to a smart contract -pub async fn run_db_migration_test() { - info!("Starting db migration test"); - - info!("Waiting to deploy contracts"); - let althea_db_addr = deploy_contracts().await; - info!("DB addr is {}", althea_db_addr); - - info!("Starting postrgresql db"); - start_postgres(); - - let conn = get_database_connection(DB_URI.to_string()).expect("Please fix db path"); - - let num_clients = 10; - // Add a bunch of dummy clients to the db to migrate - add_dummy_clients_to_db(num_clients, &conn); - - thread::sleep(Duration::from_secs(10)); - - info!("Run migration code"); - - let reg_server_key: PrivateKey = REGISTRATION_SERVER_KEY.parse().unwrap(); - // Start registration loop - info!("Registering user admin"); - // This request needs to be made with the state admin's key - check_and_add_user_admin( - &Web3::new(&get_eth_node(), WEB3_TIMEOUT), - althea_db_addr, - reg_server_key.to_address(), - reg_server_key, - Some(TRASACTION_TIMEOUT), - vec![], - ) - .await - .expect("Failed to add user admin!"); - - thread::sleep(Duration::from_secs(5)); - - let queue = Arc::new(SegQueue::new()); - - info!("Starting registration loop"); - let reg_loop = register_client_batch_loop(get_eth_node(), reg_server_key, queue.clone()); - - info!("Running user migration"); - start_db_migration( - DB_URI.to_string(), - get_eth_node(), - reg_server_key.to_address(), - althea_db_addr, - queue.clone(), - ) - .await - .expect("Failed to start migration!"); - - // wait for the timeout while also running the registration loop - match select( - Box::pin(validate_db_migration( - num_clients, - althea_db_addr, - reg_server_key, - )), - Box::pin(reg_loop), - ) - .await - { - Either::Left((_, _)) => info!("Successfully migrated all clients!"), - Either::Right((_, _)) => panic!("Registration loop crashed!"), - }; -} - -fn add_dummy_clients_to_db(num_of_entries: usize, conn: &PgConnection) { - for i in 0..num_of_entries { - let new_client = random_db_client(); - info!("Inserting new client {}: {}", i, new_client.wg_pubkey); - if let Err(e) = diesel::insert_into(clients) - .values(&new_client) - .execute(conn) - { - panic!("Why did a client {} insert fail? {}", i, e); - } - } -} - -fn random_db_client() -> Client { - let random_id = random_identity(); - Client { - mesh_ip: random_id.mesh_ip.to_string(), - wg_pubkey: random_id.wg_public_key.to_string(), - wg_port: 0, - eth_address: random_id.eth_address.to_string(), - internal_ip: "".to_string(), - internet_ipv6: "".to_string(), - nickname: "".to_string(), - email: "".to_string(), - phone: "".to_string(), - country: "".to_string(), - email_code: "".to_string(), - verified: true, - email_sent_time: 0, - text_sent: 0, - last_balance_warning_time: 0, - last_seen: 0, - } -} - -async fn validate_db_migration( - num_clients: usize, - althea_db_addr: Address, - reg_server_key: PrivateKey, -) { - let miner_pub_key = reg_server_key.to_address(); - let contact = Web3::new(&get_eth_node(), WEB3_TIMEOUT); - - let start = Instant::now(); - loop { - let client_vec = get_all_registered_clients(&contact, miner_pub_key, althea_db_addr).await; - if client_vec.is_err() { - if Instant::now() - start > Duration::from_secs(300) { - panic!("Failed to migrate clients after waiting for 5 mins"); - } - error!("No clients have been registered so far, waiting..",); - thread::sleep(Duration::from_secs(10)); - } else if let Ok(client_list) = client_vec { - if client_list.len() == num_clients { - info!( - "All clients have successuflly migrated from postgresql db to smart contract!" - ); - info!("DB clients are :\n"); - for id in client_list { - info!("{}", id); - } - break; - } else { - if Instant::now() - start > Duration::from_secs(300) { - panic!( - "Failed to migrate {} clients after waiting for 5 mins. Only migrated {}", - num_clients, - client_list.len() - ); - } - error!( - "{} clients have been registered so far, waiting..", - client_list.len() - ); - thread::sleep(Duration::from_secs(10)); - } - } - } -} diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index 5c1e4cf57..488f301f0 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -7,7 +7,6 @@ extern crate log; use std::time::Duration; pub mod contract_test; -pub mod db_migration_test; pub mod debts; pub mod five_nodes; pub mod mutli_exit; diff --git a/integration_tests/src/setup_utils/database.rs b/integration_tests/src/setup_utils/database.rs deleted file mode 100644 index 57375dad1..000000000 --- a/integration_tests/src/setup_utils/database.rs +++ /dev/null @@ -1,130 +0,0 @@ -use althea_kernel_interface::run_command; -use diesel::{Connection, PgConnection}; -use log::warn; -use std::io::Write; -use std::{ - fs::File, - io::stdout, - path::Path, - thread, - time::{Duration, Instant}, -}; - -/// Starts the exit postgres instance in the native system namespace, TODO insert plumbing so that exits can reach it -pub fn start_postgres() { - const POSTGRES_USER: &str = "postgres"; - const POSTGRES_14_BIN: &str = "/usr/lib/postgresql/14/bin/postgres"; - const POSTGRES_16_BIN: &str = "/usr/lib/postgresql/16/bin/postgres"; - const INITDB_14_BIN: &str = "/usr/lib/postgresql/14/bin/initdb"; - const INITDB_16_BIN: &str = "/usr/lib/postgresql/16/bin/initdb"; - let postgres_bin = if Path::new(POSTGRES_14_BIN).exists() { - POSTGRES_14_BIN - } else if Path::new(POSTGRES_16_BIN).exists() { - POSTGRES_16_BIN - } else { - panic!("Could not find postgres binary") - }; - let initdb_bin = if Path::new(INITDB_14_BIN).exists() { - INITDB_14_BIN - } else if Path::new(INITDB_16_BIN).exists() { - INITDB_16_BIN - } else { - panic!("Could not find initdb binary") - }; - - // for this test script - const DB_URL_LOCAL: &str = "postgres://postgres@127.0.0.1/test"; - // for the rita exit instances - const POSTGRES_DATABASE_LOCATION: &str = "/var/lib/postgresql/data"; - let migration_directory_a = - Path::new("/althea_rs/integration_tests/src/setup_utils/migrations/"); - let migration_directory_b = Path::new("integration_tests/src/setup_utils/migrations/"); - let migration_directory = if migration_directory_a.exists() { - migration_directory_a - } else if migration_directory_b.exists() { - migration_directory_b - } else { - panic!("Could not find migrations directory") - }; - let postgres_pid_path: String = format!("{}/postmaster.pid", POSTGRES_DATABASE_LOCATION); - - // only init and launch if postgres has not already been started - if !Path::new(&postgres_pid_path).exists() { - // initialize the db datadir - let res = run_command( - "sudo", - &[ - "-u", - POSTGRES_USER, - initdb_bin, - "-D", - POSTGRES_DATABASE_LOCATION, - ], - ) - .unwrap(); - if !res.status.success() { - panic!("Failed to init postgres {:?}", res); - } - - // create the pg_hba.conf with auth for the 10.0.0.1 routers - let pg_hba_path = format!("{}/pg_hba.conf", POSTGRES_DATABASE_LOCATION); - let mut pg_hba = File::create(pg_hba_path).unwrap(); - let pb_hba_lines: [&str; 4] = [ - "local all all trust", - "host all all 10.0.0.1/16 trust", - "host all all 127.0.0.1/32 trust", - "host all all ::1/128 trust", - ]; - for line in pb_hba_lines { - writeln!(pg_hba, "{}", line).unwrap() - } - } - info!("Starting postgres"); - // start postgres in it's own thread, we kill it every time we startup - // so it's spawned in this context - thread::spawn(move || { - let res = run_command( - "sudo", - &[ - "-u", - POSTGRES_USER, - postgres_bin, - "-D", - POSTGRES_DATABASE_LOCATION, - ], - ) - .unwrap(); - panic!("Postgres has crashed {:?}", res); - }); - - // create connection to the now started database - let mut conn = PgConnection::establish(DB_URL_LOCAL); - const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); - let start = Instant::now(); - while let Err(e) = conn { - warn!("Waiting for db to start {:?}", e); - if Instant::now() - start > STARTUP_TIMEOUT { - panic!("Postgres did not start! {:?}", e); - } - - // reset database contents for every run, this is in the loop becuase it too must wait until the db has started - run_command("psql", &["-c", "drop database test;", "-U", POSTGRES_USER]).unwrap(); - run_command( - "psql", - &["-c", "create database test;", "-U", POSTGRES_USER], - ) - .unwrap(); - - conn = PgConnection::establish(DB_URL_LOCAL); - thread::sleep(Duration::from_millis(1000)); - } - let conn = conn.unwrap(); - - // run diesel migrations - diesel_migrations::run_pending_migrations_in_directory( - &conn, - migration_directory, - &mut stdout(), - ) - .unwrap(); -} diff --git a/integration_tests/src/setup_utils/mod.rs b/integration_tests/src/setup_utils/mod.rs index 1d81c422b..db40c8031 100644 --- a/integration_tests/src/setup_utils/mod.rs +++ b/integration_tests/src/setup_utils/mod.rs @@ -1,4 +1,3 @@ pub mod babel; -pub mod database; pub mod namespaces; pub mod rita; diff --git a/integration_tests/src/utils.rs b/integration_tests/src/utils.rs index 42fabcd29..74871076e 100644 --- a/integration_tests/src/utils.rs +++ b/integration_tests/src/utils.rs @@ -1156,7 +1156,7 @@ pub async fn validate_debt_entry( pub async fn register_all_namespaces_to_exit(namespaces: NamespaceInfo) { for r in namespaces.names.clone() { - if let NodeType::Client { exit_name } = r.node_type.clone() { + if let NodeType::Client { exit_name, .. } = r.node_type.clone() { let res = register_to_exit(r.get_name()).await; if !res { panic!("Failed to register {} to exit with {}", r.get_name(), res); diff --git a/rita_bin/Cargo.toml b/rita_bin/Cargo.toml index aea0bf303..061fc801d 100644 --- a/rita_bin/Cargo.toml +++ b/rita_bin/Cargo.toml @@ -13,10 +13,6 @@ path = "src/exit.rs" name = "rita" path = "src/client.rs" -[[bin]] -name = "contract-util" -path = "src/contract-util.rs" - [[bin]] name = "rita_extender" path = "src/extender.rs" @@ -30,7 +26,6 @@ actix-web = { workspace = true } docopt = "1.1" compressed_log = "0.5" settings = { path = "../settings" } -diesel = { version = "1.4", features = ["postgres", "r2d2"] } log = { version = "0.4", features = ["release_max_level_info"] } serde = "1.0" serde_json = "1.0" @@ -42,7 +37,6 @@ r2d2 = "0.8" ctrlc = { version = "3.4.5", features = ["termination"] } rita_common = { path = "../rita_common", default-features = false } rita_client = { path = "../rita_client", default-features = false } -rita_db_migration = { path = "../rita_db_migration", default-features = false } rita_exit = { path = "../rita_exit", default-features = false } rita_extender = { path = "../rita_extender", default-features = false } exit_trust_root = { path = "../exit_trust_root" } diff --git a/rita_bin/src/contract-util.rs b/rita_bin/src/contract-util.rs deleted file mode 100644 index fc22cc55d..000000000 --- a/rita_bin/src/contract-util.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! This binary is a set of utilities for interacting with the Althea exit infrastructure contract. This contract contains a list of -//! all clients and exits and is used to perform key exchange between both sides. This binary contains a set of utilities for interacting -//! with this contract and performing a variety of tasks. - -#![warn(clippy::all)] -#![allow(clippy::pedantic)] -#![forbid(unsafe_code)] - -use althea_types::regions::Regions; -use althea_types::ExitIdentity; -use althea_types::Identity; -use althea_types::SystemChain; -use clarity::PrivateKey; -use diesel::RunQueryDsl; -use docopt::Docopt; -use exit_trust_root::client_db::add_exits_to_registration_list; -use exit_trust_root::client_db::add_users_to_registered_list; -use exit_trust_root::client_db::get_all_registered_clients; -use exit_trust_root::register_client_batch_loop::MAX_BATCH_SIZE; -use log::{error, info}; -use rita_db_migration::{ - get_database_connection, - models::{self, Client}, - schema::clients::dsl::clients, -}; -use serde::Deserialize; -use std::collections::HashSet; -use std::{process::exit, time::Duration}; -use web30::{client::Web3, types::SendTxOption}; - -const WEB3_TIMEOUT: Duration = Duration::from_secs(15); -pub const TX_TIMEOUT: Duration = Duration::from_secs(60); -const EXIT_REGISTRATION_PORT: u16 = 4875; -const EXIT_WG_LISTEN_PORT: u16 = 59998; - -#[derive(Debug, Deserialize)] -pub struct Args { - pub cmd_migrate: bool, - pub cmd_add_exit: bool, - pub flag_dburl: String, - pub flag_address: String, - pub flag_web3url: String, - pub flag_privatekey: String, -} - -#[actix_rt::main] -async fn main() { - env_logger::Builder::default() - .filter(None, log::LevelFilter::Info) - .init(); - - let args: Args = Docopt::new(get_arg_usage()) - .and_then(|d| d.deserialize()) - .unwrap_or_else(|e| e.exit()); - - let db_url = args.flag_dburl; - let contract_addr = args - .flag_address - .parse() - .expect("Please provide a valid eth contract addr"); - let private_key: PrivateKey = args - .flag_privatekey - .parse() - .expect("Please provide a valid eth private key with funds"); - let address = private_key.to_address(); - - let web3 = Web3::new(&args.flag_web3url, WEB3_TIMEOUT); - - if args.cmd_migrate { - // get a copy of all existing clients, we do this in order to handle a potential future edgecase where more than one registration server - // is operating at a time and the same user attempts to register to more than one before the transaction can be sent. Without this check - // once a already registered user is in the queue all future transactions would fail and the server would no longer operate correctly - let all_contract_clients = - match get_all_registered_clients(&web3, address, contract_addr).await { - Ok(all_clients) => all_clients, - Err(e) => { - panic!("Failed to get list of already registered clients {:?}", e); - } - }; - - let db_conn = get_database_connection(db_url).unwrap(); - - let database_clients_list = clients.load::(&db_conn).unwrap(); - let database_clients_list = clients_to_ids(database_clients_list); - - let mut clients_to_register = Vec::new(); - for client in database_clients_list { - if !all_contract_clients.contains(&client) { - clients_to_register.push(client); - if clients_to_register.len() > MAX_BATCH_SIZE { - break; - } - } - } - // if there is no one once we filter already registered users - if clients_to_register.is_empty() { - info!("No new clients to register! Successfully exiting"); - exit(0); - } - info!( - "Starting registration of {} clients", - clients_to_register.len() - ); - - while !clients_to_register.is_empty() { - let mut register_batch = Vec::new(); - - // build a small batch to register - while register_batch.len() < MAX_BATCH_SIZE { - if let Some(client) = clients_to_register.pop() { - register_batch.push(client); - } else { - break; - } - } - - info!("Prepped user batch sending register tx"); - match add_users_to_registered_list( - &web3, - register_batch.clone(), - contract_addr, - private_key, - Some(TX_TIMEOUT), - vec![ - SendTxOption::GasPriorityFee(100000000000u128.into()), - SendTxOption::GasMaxFee(400000000000u128.into()), - ], - ) - .await - { - Ok(_) => { - info!( - "Successfully registered {} clients!", - clients_to_register.len() - ); - } - Err(e) => { - error!("Failed to register clients with {:?}, will try again!", e); - for client in register_batch { - clients_to_register.push(client); - } - } - } - } - info!("Successfully migrated all users!"); - } else if args.cmd_add_exit { - let mut xdai = HashSet::new(); - xdai.insert(SystemChain::Xdai); - let mut usa = HashSet::new(); - usa.insert(Regions::UnitedStates); - - // This command helps generate the bytes for registering a set of exits - let exits_to_register = vec![ - ExitIdentity { - mesh_ip: "fd00::2602:9000".parse().unwrap(), - wg_key: "4PsEKlDEF8gcj9oXtt3Gi+ZmaGuxBwRMxNJ/ewCZpis=" - .parse() - .unwrap(), - eth_addr: "0xdE8236B129Ae270B75DED07101727fB03C39AA5F" - .parse() - .unwrap(), - registration_port: EXIT_REGISTRATION_PORT, - wg_exit_listen_port: EXIT_WG_LISTEN_PORT, - allowed_regions: usa.clone(), - payment_types: xdai.clone(), - }, - ExitIdentity { - mesh_ip: "fd00::2602:3000".parse().unwrap(), - wg_key: "uNu3IMSgt3SY2+MvtEwjEpx45lOk7q/7sWC3ff80GXE=" - .parse() - .unwrap(), - eth_addr: "0x72d9E579f691D62aA7e0703840db6dd2fa9fAE21" - .parse() - .unwrap(), - registration_port: EXIT_REGISTRATION_PORT, - wg_exit_listen_port: EXIT_WG_LISTEN_PORT, - allowed_regions: usa, - payment_types: xdai, - }, - ]; - match add_exits_to_registration_list( - &web3, - exits_to_register.clone(), - contract_addr, - private_key, - Some(TX_TIMEOUT), - vec![ - SendTxOption::GasPriorityFee(100000000000u128.into()), - SendTxOption::GasMaxFee(400000000000u128.into()), - ], - ) - .await - { - Ok(_) => { - info!("Successfully registered {} exits!", exits_to_register.len()); - } - Err(e) => { - error!("Failed to register exits with {:?}", e); - } - } - } -} - -pub fn get_arg_usage() -> String { - "Usage: - contract-util migrate --dburl= --address=
--web3url= --privatekey= - contract-util add-exit --address=
--web3url= --privatekey= - contract-util (-h | --help) - -Options: - -u, --dburl= Postgresql db url - -a, --address=
Smart Contract address - -w, --web3url= Web3 url - -p, --privatekey= The contract state admin private key - -About: - Utilities for interacting with the Althea exit database contract".to_string() -} - -fn clients_to_ids(client_list: Vec) -> Vec { - let mut res = Vec::new(); - for c in client_list { - res.push(Identity { - mesh_ip: match c.mesh_ip.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} mesh ip! with {}", c, e); - continue; - } - }, - eth_address: match c.eth_address.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} eth addr! with {}", c, e); - continue; - } - }, - wg_public_key: match c.wg_pubkey.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} wg key! with {}", c, e); - continue; - } - }, - nickname: None, - }); - } - res -} diff --git a/rita_db_migration/Cargo.toml b/rita_db_migration/Cargo.toml deleted file mode 100644 index 605817794..000000000 --- a/rita_db_migration/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "rita_db_migration" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -r2d2 = "0.8" -diesel = { version = "1.4", features = ["postgres", "r2d2"] } -log = { version = "0.4", features = ["release_max_level_info"] } -dotenv = "0.15" -althea_types = { path = "../althea_types" } -serde = "1.0" -serde_derive = "1.0" -serde_json = "1.0" -exit_trust_root = { path = "../exit_trust_root" } -clarity = "1.4" -web30 = "1.4" -crossbeam = "0.8" \ No newline at end of file diff --git a/rita_db_migration/src/error.rs b/rita_db_migration/src/error.rs deleted file mode 100644 index 08e4d7c75..000000000 --- a/rita_db_migration/src/error.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::{ - error::Error, - fmt::{Display, Formatter, Result as FmtResult}, -}; - -#[derive(Debug)] -pub enum RitaDBMigrationError { - MiscStringError(String), -} - -impl Display for RitaDBMigrationError { - fn fmt(&self, f: &mut Formatter) -> FmtResult { - match self { - RitaDBMigrationError::MiscStringError(a) => write!(f, "{a}",), - } - } -} - -impl Error for RitaDBMigrationError {} diff --git a/rita_db_migration/src/lib.rs b/rita_db_migration/src/lib.rs deleted file mode 100644 index 8e1e5a038..000000000 --- a/rita_db_migration/src/lib.rs +++ /dev/null @@ -1,145 +0,0 @@ -#[macro_use] -extern crate log; -#[macro_use] -extern crate diesel; -#[macro_use] -extern crate serde_derive; - -pub mod error; -pub mod models; -pub mod schema; - -use std::{collections::HashSet, sync::Arc, time::Duration}; - -use crate::schema::clients::dsl::clients; -use althea_types::Identity; -use clarity::Address; -use crossbeam::queue::SegQueue; -use diesel::{r2d2::ConnectionManager, PgConnection, RunQueryDsl}; -use error::RitaDBMigrationError; -use exit_trust_root::{ - client_db::get_all_registered_clients, register_client_batch_loop::RegistrationRequest, -}; -use models::Client; -use r2d2::PooledConnection; -use web30::client::Web3; - -const WEB3_TIMEOUT: Duration = Duration::from_secs(60); - -pub async fn start_db_migration( - db_url: String, - web3_url: String, - requester_address: Address, - db_addr: Address, - reg_queue: Arc>, -) -> Result<(), RitaDBMigrationError> { - // Validate that db_url and contract_addr are valid - if !(db_url.contains("postgres://") - || db_url.contains("postgresql://") - || db_url.contains("psql://")) - { - panic!("You must provide a valid postgressql database uri!"); - } - - let db_conn = get_database_connection(db_url)?; - - if let Ok(clients_list) = clients.load::(&db_conn) { - info!( - "Recieved a valid client list with {} entries", - clients_list.len() - ); - - let contact = Web3::new(&web3_url, WEB3_TIMEOUT); - add_clients_to_reg_queue( - clients_list, - &contact, - requester_address, - db_addr, - reg_queue, - ) - .await - } else { - return Err(RitaDBMigrationError::MiscStringError( - "Unable to get db clients".to_string(), - )); - } - - Ok(()) -} - -async fn add_clients_to_reg_queue( - client_list: Vec, - contact: &Web3, - requester_address: Address, - contract: Address, - reg_queue: Arc>, -) { - let existing_users: HashSet = - match get_all_registered_clients(contact, requester_address, contract).await { - Ok(a) => HashSet::from_iter(a.iter().cloned()), - Err(e) => { - error!( - "Failed to get a list of existing users with {}!. Trying to add all users", - e - ); - HashSet::new() - } - }; - - for c in client_list { - let id = Identity { - mesh_ip: match c.mesh_ip.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} mesh ip! with {}", c, e); - continue; - } - }, - eth_address: match c.eth_address.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} eth addr! with {}", c, e); - continue; - } - }, - wg_public_key: match c.wg_pubkey.parse() { - Ok(a) => a, - Err(e) => { - error!("Cannot parse client {:?} wg key! with {}", c, e); - continue; - } - }, - nickname: None, - }; - - if !existing_users.contains(&id) { - info!("Adding user {}", id.mesh_ip); - reg_queue.push(RegistrationRequest { - identity: id, - contract, - }); - } else { - warn!("User {} already exists!", id.mesh_ip); - } - } -} - -pub fn get_database_connection( - db_url: String, -) -> Result>, RitaDBMigrationError> { - let manager = ConnectionManager::new(db_url); - let pool = r2d2::Pool::builder() - .max_size(1) - .build(manager) - .expect("Failed to create pool."); - - match pool.try_get() { - Some(connection) => Ok(connection), - None => { - error!("No available db connection!"); - Err(RitaDBMigrationError::MiscStringError( - "No Database connection available!".to_string(), - )) - } - } -} diff --git a/rita_db_migration/src/models.rs b/rita_db_migration/src/models.rs deleted file mode 100644 index fb9c3538b..000000000 --- a/rita_db_migration/src/models.rs +++ /dev/null @@ -1,37 +0,0 @@ -#![allow(clippy::extra_unused_lifetimes)] -use crate::schema::{assigned_ips, clients}; - -#[derive(Queryable, Serialize, Deserialize, Debug, Insertable, Clone, Default)] -#[table_name = "clients"] -pub struct Client { - pub mesh_ip: String, - pub wg_pubkey: String, - pub wg_port: i32, - pub eth_address: String, - pub internal_ip: String, - pub internet_ipv6: String, - pub nickname: String, - pub email: String, - pub phone: String, - pub country: String, - pub email_code: String, - pub verified: bool, - pub email_sent_time: i64, - pub text_sent: i32, - pub last_seen: i64, - pub last_balance_warning_time: i64, -} - -/// This struct holds information about the ipv6 subnets being assigned to clients who connect. -/// The vector available subnets is a stack that has a list of available subnets to use. This stack gets populated whenever -/// a client gets removed from the database. It is stored as a string of indecies, for example, "1,24,36" -/// The iterative index stores the index at which we assign a subnet to a client -/// For example, if our exit subnet is fd00::1000/120 and our client subnets are /124, index 0 represents -/// fd00::1000/124 index 1 represents fd00::1010/124, 2 is fd00::1120/124 etc... -#[derive(Queryable, Serialize, Deserialize, Debug, Insertable, Clone, Default)] -#[table_name = "assigned_ips"] -pub struct AssignedIps { - pub subnet: String, - pub available_subnets: String, - pub iterative_index: i64, -} diff --git a/rita_db_migration/src/schema.rs b/rita_db_migration/src/schema.rs deleted file mode 100644 index d8470b758..000000000 --- a/rita_db_migration/src/schema.rs +++ /dev/null @@ -1,28 +0,0 @@ -table! { - clients (mesh_ip) { - mesh_ip -> Varchar, - wg_pubkey -> Varchar, - wg_port -> Int4, - eth_address -> Varchar, - internal_ip -> Varchar, - internet_ipv6 -> Varchar, - nickname -> Varchar, - email -> Varchar, - phone -> Varchar, - country -> Varchar, - email_code -> Varchar, - verified -> Bool, - email_sent_time -> Int8, - text_sent -> Int4, - last_seen -> Int8, - last_balance_warning_time -> Int8, - } -} - -table! { - assigned_ips (subnet) { - subnet -> Varchar, - available_subnets -> Varchar, - iterative_index -> Int8, - } -} diff --git a/scripts/integration_tests/all-up-test-ci.sh b/scripts/integration_tests/all-up-test-ci.sh index 9fabfaefa..b1cd9ccb7 100644 --- a/scripts/integration_tests/all-up-test-ci.sh +++ b/scripts/integration_tests/all-up-test-ci.sh @@ -12,11 +12,6 @@ sudo apt install -y sudo \ jq \ vim \ netcat-traditional \ - default-libmysqlclient-dev \ - libsqlite3-dev \ - postgresql-client-14 \ - postgresql-server-dev-14 \ - libpq-dev \ python3-pip \ bridge-utils \ wireguard \ diff --git a/test_runner/src/main.rs b/test_runner/src/main.rs index 9dadf0a9f..869ddf059 100644 --- a/test_runner/src/main.rs +++ b/test_runner/src/main.rs @@ -1,5 +1,4 @@ use integration_tests::contract_test::run_altheadb_contract_test; -use integration_tests::db_migration_test::run_db_migration_test; use integration_tests::debts::run_debts_test; /// Binary crate for actually running the integration tests use integration_tests::five_nodes::run_five_node_test_scenario; @@ -50,8 +49,6 @@ async fn main() { run_multi_exit_test().await } else if test_type == "CONTRACT_TEST" { run_altheadb_contract_test().await - } else if test_type == "MIGRATION_TEST" || test_type == "DB_MIGRATION" { - run_db_migration_test().await } else { panic!("Error unknown test type {}!", test_type); }