Adding a test for db migration from postgresql to smart contract

.github/workflows/rust.yml

@@ -135,4 +135,13 @@ jobs:
       - name: Install Wireguard
         run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard
       - name: Run integration test
-        run: bash scripts/integration_tests/all-up-test.sh CONTRACT_TEST
+        run: bash scripts/integration_tests/all-up-test.sh CONTRACT_TEST
+  integration-test-db-migration:
+    needs: integration-test-five-nodes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Wireguard
+        run: sudo apt-get update && sudo apt install -y wireguard linux-source linux-headers-$(uname -r) build-essential && sudo modprobe wireguard
+      - name: Run integration test
+        run: bash scripts/integration_tests/all-up-test.sh MIGRATION_TEST

integration_tests/Cargo.toml

@@ -22,6 +22,7 @@ rita_client = { path = "../rita_client", features = ["dev_env"] }
 rita_common = { path = "../rita_common", features = ["integration_test"] }
 rita_exit = { path = "../rita_exit", features = ["dev_env"] }
 rita_client_registration = { path = "../rita_client_registration" }
+rita_db_migration = { path = "../rita_db_migration" }
 ctrlc = { version = "3.2.1", features = ["termination"] }
 diesel = { version = "1.4", features = ["postgres", "r2d2"] }
 diesel_migrations = { version = "1.4", features = ["postgres"] }

integration_tests/src/db_migration_test.rs

@@ -0,0 +1,141 @@
+use std::{
+    thread,
+    time::{Duration, Instant},
+};
+
+use clarity::{Address, PrivateKey};
+use diesel::{PgConnection, RunQueryDsl};
+use log::{error, info};
+use rita_client_registration::{
+    client_db::get_all_regsitered_clients, register_client_batch_loop::register_client_batch_loop,
+};
+use rita_common::usage_tracker::tests::test::random_identity;
+use rita_db_migration::{
+    get_database_connection, models::Client, schema::clients::dsl::clients, start_db_migration,
+};
+use web30::client::Web3;
+
+use crate::{
+    payments_eth::{get_miner_key, WEB3_TIMEOUT},
+    setup_utils::database::start_postgres,
+    utils::{deploy_contracts, get_eth_node},
+};
+
+pub const DB_URI: &str = "postgres://postgres@localhost/test";
+
+/// This tests the rita_db_migration binary and veries that clients actually migrate from a postgresql db
+/// to a smart contract
+pub async fn run_db_migration_test() {
+    info!("Starting db migration test");
+
+    info!("Waiting to deploy contracts");
+    let althea_db_addr = deploy_contracts().await;
+    info!("DB addr is {}", althea_db_addr);
+
+    info!("Starting postrgresql db");
+    start_postgres();
+
+    let conn = get_database_connection(DB_URI.to_string()).expect("Please fix db path");
+
+    let num_clients = 10;
+    // Add a bunch of dummy clients to the db to migrate
+    add_dummy_clients_to_db(num_clients, &conn);
+
+    thread::sleep(Duration::from_secs(10));
+
+    info!("Run migration code");
+
+    let miner_private_key: PrivateKey = get_miner_key();
+    // Start registration loop
+    register_client_batch_loop(get_eth_node(), althea_db_addr, miner_private_key);
+
+    match start_db_migration(DB_URI.to_string()) {
+        Ok(_) => println!("Successfully migrated all clients!"),
+        Err(e) => println!("Failed to migrate clients with {}", e),
+    }
+
+    info!("Waiting for register loop to migrate all clients");
+    thread::sleep(Duration::from_secs(10));
+
+    validate_db_migration(num_clients, althea_db_addr, miner_private_key).await;
+}
+
+fn add_dummy_clients_to_db(num_of_entries: usize, conn: &PgConnection) {
+    for i in 0..num_of_entries {
+        let new_client = random_db_client();
+        info!("Inserting new client {}: {}", i, new_client.wg_pubkey);
+        if let Err(e) = diesel::insert_into(clients)
+            .values(&new_client)
+            .execute(conn)
+        {
+            panic!("Why did a client {} insert fail? {}", i, e);
+        }
+    }
+}
+
+fn random_db_client() -> Client {
+    let random_id = random_identity();
+    Client {
+        mesh_ip: random_id.mesh_ip.to_string(),
+        wg_pubkey: random_id.wg_public_key.to_string(),
+        wg_port: 0,
+        eth_address: random_id.eth_address.to_string(),
+        internal_ip: "".to_string(),
+        internet_ipv6: "".to_string(),
+        nickname: "".to_string(),
+        email: "".to_string(),
+        phone: "".to_string(),
+        country: "".to_string(),
+        email_code: "".to_string(),
+        verified: true,
+        email_sent_time: 0,
+        text_sent: 0,
+        last_balance_warning_time: 0,
+        last_seen: 0,
+    }
+}
+
+async fn validate_db_migration(
+    num_clients: usize,
+    althea_db_addr: Address,
+    miner_private_key: PrivateKey,
+) {
+    let miner_pub_key = miner_private_key.to_address();
+    let contact = Web3::new(&get_eth_node(), WEB3_TIMEOUT);
+
+    let start = Instant::now();
+    loop {
+        let client_vec = get_all_regsitered_clients(&contact, miner_pub_key, althea_db_addr).await;
+        if client_vec.is_err() {
+            if Instant::now() - start > Duration::from_secs(300) {
+                panic!("Failed to migrate clients after waiting for 5 mins");
+            }
+            error!("No clients have been registered so far, waiting..",);
+            thread::sleep(Duration::from_secs(10));
+        } else if let Ok(client_list) = client_vec {
+            if client_list.len() == num_clients {
+                info!(
+                    "All clients have successuflly migrated from postgresql db to smart contract!"
+                );
+                info!("DB clients are :\n");
+                for id in client_list {
+                    info!("{}", id);
+                }
+                break;
+            } else {
+                if Instant::now() - start > Duration::from_secs(300) {
+                    panic!(
+                        "Failed to migrate {} clients after waiting for 5 mins. Only migrated {}",
+                        num_clients,
+                        client_list.len()
+                    );
+                }
+                error!(
+                    "{} clients have been registered so far, waiting..",
+                    client_list.len()
+                );
+                thread::sleep(Duration::from_secs(10));
+            }
+        }
+    }
+}

integration_tests/src/lib.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 
 pub mod config;
 pub mod contract_test;
+pub mod db_migration_test;
 pub mod debts;
 pub mod five_nodes;
 pub mod mutli_exit;

integration_tests/src/setup_utils/database.rs

@@ -19,7 +19,7 @@ pub fn start_postgres() {
     const DB_URL_LOCAL: &str = "postgres://[email protected]/test";
     // for the rita exit instances
     const POSTGRES_DATABASE_LOCATION: &str = "/var/lib/postgresql/data";
-    let migration_directory = Path::new("/althea_rs/exit_db/migrations/");
+    let migration_directory = Path::new("/althea_rs/integration_tests/src/setup_utils/migrations/");
     let postgres_pid_path: String = format!("{}/postmaster.pid", POSTGRES_DATABASE_LOCATION);
 
     // only init and launch if postgres has not already been started

integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/down.sql

@@ -0,0 +1,6 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
+DROP FUNCTION IF EXISTS diesel_set_updated_at();

integration_tests/src/setup_utils/migrations/00000000000000_diesel_initial_setup/up.sql

@@ -0,0 +1,36 @@
+-- This file was automatically created by Diesel to setup helper functions
+-- and other internal bookkeeping. This file is safe to edit, any future
+-- changes will be added to existing projects as new migrations.
+
+
+
+
+-- Sets up a trigger for the given table to automatically set a column called
+-- `updated_at` whenever the row is modified (unless `updated_at` was included
+-- in the modified columns)
+--
+-- # Example
+--
+-- ```sql
+-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
+--
+-- SELECT diesel_manage_updated_at('users');
+-- ```
+CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
+BEGIN
+    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
+                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
+BEGIN
+    IF (
+        NEW IS DISTINCT FROM OLD AND
+        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
+    ) THEN
+        NEW.updated_at := current_timestamp;
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;

integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE clients;

integration_tests/src/setup_utils/migrations/2019-02-22-214628_rita-setup/up.sql

@@ -0,0 +1,18 @@
+CREATE TABLE clients
+(
+    mesh_ip varchar(40) CONSTRAINT firstkey PRIMARY KEY,
+    wg_pubkey varchar(44) NOT NULL,
+    wg_port integer NOT NULL,
+    eth_address varchar(64) NOT NULL,
+    internal_ip varchar(42) NOT NULL,
+    nickname varchar(32) NOT NULL,
+    email varchar(512) NOT NULL,
+    phone varchar(32) NOT NULL,
+    country varchar(8) NOT NULL,
+    email_code varchar(16) NOT NULL,
+    verified boolean DEFAULT FALSE NOT NULL,
+    email_sent_time bigint DEFAULT 0 NOT NULL,
+    text_sent integer DEFAULT 0 NOT NULL,
+    last_seen bigint DEFAULT 0 NOT NULL,
+    last_balance_warning_time bigint DEFAULT 0 NOT NULL
+);

integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/down.sql

@@ -0,0 +1,5 @@
+-- This file should undo anything in `up.sql`
+ALTER TABLE clients
+    DROP COLUMN internet_ipv6
+;
+DROP TABLE assigned_ips; 

integration_tests/src/setup_utils/migrations/2023-09-26-180549_ipv6_migration/up.sql

@@ -0,0 +1,10 @@
+-- Your SQL goes here
+ALTER TABLE clients
+    ADD COLUMN internet_ipv6 varchar(132) NOT NULL DEFAULT ''
+;
+CREATE TABLE assigned_ips
+(
+    subnet varchar(132) CONSTRAINT secondkey PRIMARY KEY,
+    available_subnets varchar(512) NOT NULL,
+    iterative_index bigint DEFAULT 0 NOT NULL
+); 

test_runner/src/main.rs

@@ -2,6 +2,7 @@ use integration_tests::config::{
     generate_exit_config_file, generate_rita_config_file, CONFIG_FILE_PATH, EXIT_CONFIG_PATH,
 };
 use integration_tests::contract_test::run_altheadb_contract_test;
+use integration_tests::db_migration_test::run_db_migration_test;
 use integration_tests::debts::run_debts_test;
 /// Binary crate for actually running the integration tests
 use integration_tests::five_nodes::run_five_node_test_scenario;
@@ -55,6 +56,8 @@ async fn main() {
             run_multi_exit_test().await
         } else if test_type == "CONTRACT_TEST" {
             run_altheadb_contract_test().await
+        } else if test_type == "MIGRATION_TEST" {
+            run_db_migration_test().await
         } else {
             panic!("Error unknown test type {}!", test_type);
         }