diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b469455ad..be7e8e62a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -135,4 +135,13 @@ jobs: - name: Install Wireguard run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard - name: Run integration test - run: bash scripts/integration_tests/all-up-test.sh CONTRACT_TEST \ No newline at end of file + run: bash scripts/integration_tests/all-up-test.sh CONTRACT_TEST + integration-test-db-migration: + needs: integration-test-five-nodes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Wireguard + run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard + - name: Run integration test + run: bash scripts/integration_tests/all-up-test.sh MIGRATION_TEST \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index ed35ec842..430e7578e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1621,6 +1621,7 @@ dependencies = [ "rita_client", "rita_client_registration", "rita_common", + "rita_db_migration", "rita_exit", "settings", "web30", diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 4fb51fb48..7efc0d96e 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -22,6 +22,7 @@ rita_client = { path = "../rita_client", features = ["dev_env"] } rita_common = { path = "../rita_common", features = ["integration_test"] } rita_exit = { path = "../rita_exit", features = ["dev_env"] } rita_client_registration = { path = "../rita_client_registration" } +rita_db_migration = { path = "../rita_db_migration" } ctrlc = { version = "3.2.1", features = ["termination"] } diesel = { version = "1.4", features = ["postgres", "r2d2"] } diesel_migrations = { version = "1.4", features = ["postgres"] } diff --git a/integration_tests/src/db_migration_test.rs b/integration_tests/src/db_migration_test.rs new file mode 100644 index 000000000..205ac9e59 --- /dev/null +++ b/integration_tests/src/db_migration_test.rs @@ -0,0 +1,141 @@ +use std::{ + thread, + time::{Duration, Instant}, +}; + +use clarity::{Address, PrivateKey}; +use diesel::{PgConnection, RunQueryDsl}; +use log::{error, info}; +use rita_client_registration::{ + client_db::get_all_regsitered_clients, register_client_batch_loop::register_client_batch_loop, +}; +use rita_common::usage_tracker::tests::test::random_identity; +use rita_db_migration::{ + get_database_connection, models::Client, schema::clients::dsl::clients, start_db_migration, +}; +use web30::client::Web3; + +use crate::{ + payments_eth::{get_miner_key, WEB3_TIMEOUT}, + setup_utils::database::start_postgres, + utils::{deploy_contracts, get_eth_node}, +}; + +pub const DB_URI: &str = "postgres://postgres@localhost/test"; + +/// This tests the rita_db_migration binary and veries that clients actually migrate from a postgresql db +/// to a smart contract +pub async fn run_db_migration_test() { + info!("Starting db migration test"); + + info!("Waiting to deploy contracts"); + let althea_db_addr = deploy_contracts().await; + info!("DB addr is {}", althea_db_addr); + + info!("Starting postrgresql db"); + start_postgres(); + + let conn = get_database_connection(DB_URI.to_string()).expect("Please fix db path"); + + let num_clients = 10; + // Add a bunch of dummy clients to the db to migrate + add_dummy_clients_to_db(num_clients, &conn); + + thread::sleep(Duration::from_secs(10)); + + info!("Run migration code"); + + let miner_private_key: PrivateKey = get_miner_key(); + // Start registration loop + register_client_batch_loop(get_eth_node(), althea_db_addr, miner_private_key); + + match start_db_migration(DB_URI.to_string()) { + Ok(_) => println!("Successfully migrated all clients!"), + Err(e) => println!("Failed to migrate clients with {}", e), + } + + info!("Waiting for register loop to migrate all clients"); + thread::sleep(Duration::from_secs(10)); + + validate_db_migration(num_clients, althea_db_addr, miner_private_key).await; +} + +fn add_dummy_clients_to_db(num_of_entries: usize, conn: &PgConnection) { + for i in 0..num_of_entries { + let new_client = random_db_client(); + info!("Inserting new client {}: {}", i, new_client.wg_pubkey); + if let Err(e) = diesel::insert_into(clients) + .values(&new_client) + .execute(conn) + { + panic!("Why did a client {} insert fail? {}", i, e); + } + } +} + +fn random_db_client() -> Client { + let random_id = random_identity(); + Client { + mesh_ip: random_id.mesh_ip.to_string(), + wg_pubkey: random_id.wg_public_key.to_string(), + wg_port: 0, + eth_address: random_id.eth_address.to_string(), + internal_ip: "".to_string(), + internet_ipv6: "".to_string(), + nickname: "".to_string(), + email: "".to_string(), + phone: "".to_string(), + country: "".to_string(), + email_code: "".to_string(), + verified: true, + email_sent_time: 0, + text_sent: 0, + last_balance_warning_time: 0, + last_seen: 0, + } +} + +async fn validate_db_migration( + num_clients: usize, + althea_db_addr: Address, + miner_private_key: PrivateKey, +) { + let miner_pub_key = miner_private_key.to_address(); + let contact = Web3::new(&get_eth_node(), WEB3_TIMEOUT); + + let start = Instant::now(); + loop { + let client_vec = get_all_regsitered_clients(&contact, miner_pub_key, althea_db_addr).await; + if client_vec.is_err() { + if Instant::now() - start > Duration::from_secs(300) { + panic!("Failed to migrate clients after waiting for 5 mins"); + } + error!("No clients have been registered so far, waiting..",); + thread::sleep(Duration::from_secs(10)); + } else if let Ok(client_list) = client_vec { + if client_list.len() == num_clients { + info!( + "All clients have successuflly migrated from postgresql db to smart contract!" + ); + info!("DB clients are :\n"); + for id in client_list { + info!("{}", id); + } + break; + } else { + if Instant::now() - start > Duration::from_secs(300) { + panic!( + "Failed to migrate {} clients after waiting for 5 mins. Only migrated {}", + num_clients, + client_list.len() + ); + } + error!( + "{} clients have been registered so far, waiting..", + client_list.len() + ); + thread::sleep(Duration::from_secs(10)); + } + } + } +} diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index b0e86e794..a1365d60e 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -5,6 +5,7 @@ use std::time::Duration; pub mod config; pub mod contract_test; +pub mod db_migration_test; pub mod debts; pub mod five_nodes; pub mod mutli_exit; diff --git a/integration_tests/src/setup_utils/database.rs b/integration_tests/src/setup_utils/database.rs index 5acf7851c..76089c187 100644 --- a/integration_tests/src/setup_utils/database.rs +++ b/integration_tests/src/setup_utils/database.rs @@ -19,7 +19,7 @@ pub fn start_postgres() { const DB_URL_LOCAL: &str = "postgres://postgres@127.0.0.1/test"; // for the rita exit instances const POSTGRES_DATABASE_LOCATION: &str = "/var/lib/postgresql/data"; - let migration_directory = Path::new("/althea_rs/exit_db/migrations/"); + let migration_directory = Path::new("/althea_rs/integration_tests/src/setup_utils/migrations/"); let postgres_pid_path: String = format!("{}/postmaster.pid", POSTGRES_DATABASE_LOCATION); // only init and launch if postgres has not already been started diff --git a/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/down.sql b/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 000000000..a9f526091 --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/up.sql b/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 000000000..d68895b1a --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/down.sql b/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/down.sql new file mode 100644 index 000000000..036a199de --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE clients; \ No newline at end of file diff --git a/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/up.sql b/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/up.sql new file mode 100644 index 000000000..5e2c70792 --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/up.sql @@ -0,0 +1,18 @@ +CREATE TABLE clients +( + mesh_ip varchar(40) CONSTRAINT firstkey PRIMARY KEY, + wg_pubkey varchar(44) NOT NULL, + wg_port integer NOT NULL, + eth_address varchar(64) NOT NULL, + internal_ip varchar(42) NOT NULL, + nickname varchar(32) NOT NULL, + email varchar(512) NOT NULL, + phone varchar(32) NOT NULL, + country varchar(8) NOT NULL, + email_code varchar(16) NOT NULL, + verified boolean DEFAULT FALSE NOT NULL, + email_sent_time bigint DEFAULT 0 NOT NULL, + text_sent integer DEFAULT 0 NOT NULL, + last_seen bigint DEFAULT 0 NOT NULL, + last_balance_warning_time bigint DEFAULT 0 NOT NULL +); \ No newline at end of file diff --git a/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/down.sql b/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/down.sql new file mode 100644 index 000000000..e42354d99 --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/down.sql @@ -0,0 +1,5 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE clients + DROP COLUMN internet_ipv6 +; +DROP TABLE assigned_ips; \ No newline at end of file diff --git a/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/up.sql b/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/up.sql new file mode 100644 index 000000000..cd3982421 --- /dev/null +++ b/integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/up.sql @@ -0,0 +1,10 @@ +-- Your SQL goes here +ALTER TABLE clients + ADD COLUMN internet_ipv6 varchar(132) NOT NULL DEFAULT '' +; +CREATE TABLE assigned_ips +( + subnet varchar(132) CONSTRAINT secondkey PRIMARY KEY, + available_subnets varchar(512) NOT NULL, + iterative_index bigint DEFAULT 0 NOT NULL +); \ No newline at end of file diff --git a/test_runner/src/main.rs b/test_runner/src/main.rs index 127a6277f..2159892c0 100644 --- a/test_runner/src/main.rs +++ b/test_runner/src/main.rs @@ -2,6 +2,7 @@ use integration_tests::config::{ generate_exit_config_file, generate_rita_config_file, CONFIG_FILE_PATH, EXIT_CONFIG_PATH, }; use integration_tests::contract_test::run_altheadb_contract_test; +use integration_tests::db_migration_test::run_db_migration_test; use integration_tests::debts::run_debts_test; /// Binary crate for actually running the integration tests use integration_tests::five_nodes::run_five_node_test_scenario; @@ -55,6 +56,8 @@ async fn main() { run_multi_exit_test().await } else if test_type == "CONTRACT_TEST" { run_altheadb_contract_test().await + } else if test_type == "MIGRATION_TEST" { + run_db_migration_test().await } else { panic!("Error unknown test type {}!", test_type); }