diff --git a/src/main.rs b/src/main.rs index c2e93c80..92bfcdf2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,7 +13,6 @@ async fn main() -> Result<(), ValveError> { // TODO: Use a more powerful command-line parser library that can automatically take care of // things like mutually exclusive options, since argparse doesn't seem to be able to do it. - let mut ad_hoc = false; // TODO: Remove the ad_hoc parameter before merging this PR. let mut verbose = false; let mut api_test = false; let mut dump_config = false; @@ -35,10 +34,6 @@ async fn main() -> Result<(), ValveError> { // this block limits scope of borrows by ap.refer() method let mut ap = ArgumentParser::new(); ap.set_description(r#"Valve is a lightweight validation engine written in rust."#); - - // TODO: Remove the ad_hoc parameter before merging this PR. - ap.refer(&mut ad_hoc) - .add_option(&["--ad_hoc"], StoreTrue, r#"Do something ad hoc."#); ap.refer(&mut verbose).add_option( &["--verbose"], StoreTrue, @@ -144,8 +139,6 @@ async fn main() -> Result<(), ValveError> { let advice = format!("Run `{} --help` for command line usage.", program_name); let mutually_exclusive_options = vec![ - // TODO: Remove the ad_hoc parameter before merging this PR. - ad_hoc, api_test, dump_config, dump_schema, @@ -181,11 +174,7 @@ async fn main() -> Result<(), ValveError> { process::exit(1); } - // TODO: Remove the ad_hoc parameter before merging this PR. - if ad_hoc { - let valve = Valve::build(&source, &destination, verbose, initial_load).await?; - valve.save_all_tables(&None)?; - } else if api_test { + if api_test { run_api_tests(&source, &destination).await?; } else if save_all || save != "" { let valve = Valve::build(&source, &destination, verbose, initial_load).await?; @@ -204,13 +193,11 @@ async fn main() -> Result<(), ValveError> { } } else if dump_config { let valve = Valve::build(&source, &destination, verbose, initial_load).await?; - // TODO: Somehow convert this output to JSON. We will likely have to rewrite the display() - // functions for the structs involved. Note that this is required for the - // test/generate_random_test_data.py to work. - println!("{:#?}", valve); + println!("{}", valve.config); } else if dump_schema { let valve = Valve::build(&source, &destination, verbose, initial_load).await?; - valve.dump_schema().await?; + let schema = valve.dump_schema().await?; + println!("{}", schema); } else if table_order { let valve = Valve::build(&source, &destination, verbose, initial_load).await?; let sorted_table_list = valve.get_sorted_table_list(false); diff --git a/src/valve.rs b/src/valve.rs index ab8418ca..f6bf2846 100644 --- a/src/valve.rs +++ b/src/valve.rs @@ -22,12 +22,13 @@ use indexmap::IndexMap; use indoc::indoc; use itertools::Itertools; use regex::Regex; +use serde::Serialize; use serde_json::{json, Value as SerdeValue}; use sqlx::{ any::{AnyKind, AnyPool, AnyRow}, query as sqlx_query, Row, ValueRef, }; -use std::{collections::HashMap, fs::File, path::Path}; +use std::{collections::HashMap, fmt, fs::File, path::Path}; /// Alias for [serde_json::Map](..//serde_json/struct.Map.html). // Note: serde_json::Map is @@ -72,7 +73,7 @@ struct _ValveChange { pub _message: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveSpecialConfig { pub column: String, pub datatype: String, @@ -80,7 +81,7 @@ pub struct ValveSpecialConfig { pub table: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveTableConfig { pub table: String, pub table_type: String, @@ -90,7 +91,7 @@ pub struct ValveTableConfig { pub column_order: Vec, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveColumnConfig { pub table: String, pub column: String, @@ -101,7 +102,7 @@ pub struct ValveColumnConfig { pub nulltype: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveDatatypeConfig { pub html_type: String, pub sql_type: String, @@ -113,7 +114,7 @@ pub struct ValveDatatypeConfig { pub transform: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveRuleConfig { pub description: String, pub level: String, @@ -124,13 +125,13 @@ pub struct ValveRuleConfig { pub when_condition: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveTreeConstraint { pub child: String, pub parent: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveUnderConstraint { pub column: String, pub ttable: String, @@ -138,7 +139,7 @@ pub struct ValveUnderConstraint { pub value: SerdeValue, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveForeignConstraint { pub table: String, pub column: String, @@ -146,7 +147,7 @@ pub struct ValveForeignConstraint { pub fcolumn: String, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveConstraintConfig { // Note that primary would be better as HashMap, since it is not possible to // have more than one primary key per table, but the below reflects the current implementation @@ -159,7 +160,7 @@ pub struct ValveConstraintConfig { pub under: HashMap>, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize)] pub struct ValveConfig { pub special: ValveSpecialConfig, pub table: HashMap, @@ -168,6 +169,16 @@ pub struct ValveConfig { pub constraint: ValveConstraintConfig, } +impl fmt::Display for ValveConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let json = serde_json::to_string(&self).expect(&format!( + "Unable to render Valve configuration: {:?} as JSON.", + self + )); + write!(f, "{}", json) + } +} + /// Main entrypoint for the Valve API. #[derive(Clone, Debug)] pub struct Valve { @@ -782,14 +793,15 @@ impl Valve { } /// Writes the database schema to stdout. - pub async fn dump_schema(&self) -> Result<(), ValveError> { + pub async fn dump_schema(&self) -> Result { let setup_statements = self.get_setup_statements().await?; + let mut output = String::from(""); for table in self.get_sorted_table_list(false) { let table_statements = setup_statements.get(table).unwrap(); - let output = String::from(table_statements.join("\n")); - println!("{}\n", output); + let table_output = String::from(table_statements.join("\n")); + output.push_str(&table_output); } - Ok(()) + Ok(output) } /// Create all configured database tables and views if they do not already exist as configured. diff --git a/test/generate_random_test_data.py b/test/generate_random_test_data.py index 87008651..661595bf 100755 --- a/test/generate_random_test_data.py +++ b/test/generate_random_test_data.py @@ -15,7 +15,7 @@ def get_special_tables(config): - return [k for k, v in config["special"].items() if v is not None] + return ["message", "history"] + [k for k, v in config["special"].items() if v is not None] def get_table_columns(config, table): @@ -35,15 +35,15 @@ def get_column_datatype(config, table, column): def get_foreign_key(config, table, column): - return [f for f in config["constraints"]["foreign"][table] if f["column"] == column][0] + return [f for f in config["constraint"]["foreign"][table] if f["column"] == column][0] def get_tree(config, table, column): - return [f for f in config["constraints"]["tree"][table] if f["parent"] == column][0] + return [f for f in config["constraint"]["tree"][table] if f["parent"] == column][0] def get_under(config, table, column): - return [f for f in config["constraints"]["under"][table] if f["column"] == column][0] + return [f for f in config["constraint"]["under"][table] if f["column"] == column][0] def get_value_from_prev_insert(config, prev_inserts, from_table, from_column, to_table, to_column): @@ -172,14 +172,23 @@ def main(): sys.exit(result.returncode) config = json.loads(result.stdout.decode()) + # Get the sorted list of tables to generate: + result = subprocess.run(["./valve", "--table_order", input_table], capture_output=True) + if result.returncode != 0: + error = result.stderr.decode() + output = result.stdout.decode() + if output: + error = f"{error}\n{output}" + print(f"{error}", file=sys.stderr) + sys.exit(result.returncode) + data_tables = [t.strip() for t in result.stdout.decode().split(",")] + data_tables = [t for t in data_tables if t not in get_special_tables(config)] + # This is a record of the last inserted values for each table and column. When one column # takes its values from another column, then we look here and fetch the last inserted value of # the second column. prev_inserts = {} - # The data tables to generate: - data_tables = [t for t in config["sorted_table_list"] if t not in get_special_tables(config)] - # The TSV files corresponding to each data table: tsv_files = {} for table in data_tables: