Skip to content

Commit

Permalink
print valve config as json; don't print directly in dump_schema(); re…
Browse files Browse the repository at this point in the history 
…move ad_hoc parameter
  • Loading branch information
@lmcmicu
lmcmicu committed Feb 13, 2024
1 parent 2dd98e9 commit 72eb8d1
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 39 deletions.
21 changes: 4 additions & 17 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ async fn main() -> Result<(), ValveError> {
// TODO: Use a more powerful command-line parser library that can automatically take care of
// things like mutually exclusive options, since argparse doesn't seem to be able to do it.

let mut ad_hoc = false; // TODO: Remove the ad_hoc parameter before merging this PR.
let mut verbose = false;
let mut api_test = false;
let mut dump_config = false;
Expand All @@ -35,10 +34,6 @@ async fn main() -> Result<(), ValveError> {
// this block limits scope of borrows by ap.refer() method
let mut ap = ArgumentParser::new();
ap.set_description(r#"Valve is a lightweight validation engine written in rust."#);

// TODO: Remove the ad_hoc parameter before merging this PR.
ap.refer(&mut ad_hoc)
.add_option(&["--ad_hoc"], StoreTrue, r#"Do something ad hoc."#);
ap.refer(&mut verbose).add_option(
&["--verbose"],
StoreTrue,
Expand Down Expand Up @@ -144,8 +139,6 @@ async fn main() -> Result<(), ValveError> {
let advice = format!("Run `{} --help` for command line usage.", program_name);

let mutually_exclusive_options = vec![
// TODO: Remove the ad_hoc parameter before merging this PR.
ad_hoc,
api_test,
dump_config,
dump_schema,
Expand Down Expand Up @@ -181,11 +174,7 @@ async fn main() -> Result<(), ValveError> {
process::exit(1);
}

// TODO: Remove the ad_hoc parameter before merging this PR.
if ad_hoc {
let valve = Valve::build(&source, &destination, verbose, initial_load).await?;
valve.save_all_tables(&None)?;
} else if api_test {
if api_test {
run_api_tests(&source, &destination).await?;
} else if save_all || save != "" {
let valve = Valve::build(&source, &destination, verbose, initial_load).await?;
Expand All @@ -204,13 +193,11 @@ async fn main() -> Result<(), ValveError> {
}
} else if dump_config {
let valve = Valve::build(&source, &destination, verbose, initial_load).await?;
// TODO: Somehow convert this output to JSON. We will likely have to rewrite the display()
// functions for the structs involved. Note that this is required for the
// test/generate_random_test_data.py to work.
println!("{:#?}", valve);
println!("{}", valve.config);
} else if dump_schema {
let valve = Valve::build(&source, &destination, verbose, initial_load).await?;
valve.dump_schema().await?;
let schema = valve.dump_schema().await?;
println!("{}", schema);
} else if table_order {
let valve = Valve::build(&source, &destination, verbose, initial_load).await?;
let sorted_table_list = valve.get_sorted_table_list(false);
Expand Down
42 changes: 27 additions & 15 deletions src/valve.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@ use indexmap::IndexMap;
use indoc::indoc;
use itertools::Itertools;
use regex::Regex;
use serde::Serialize;
use serde_json::{json, Value as SerdeValue};
use sqlx::{
any::{AnyKind, AnyPool, AnyRow},
query as sqlx_query, Row, ValueRef,
};
use std::{collections::HashMap, fs::File, path::Path};
use std::{collections::HashMap, fmt, fs::File, path::Path};

/// Alias for [serde_json::Map](..//serde_json/struct.Map.html)<String, [serde_json::Value](../serde_json/enum.Value.html)>.
// Note: serde_json::Map is
Expand Down Expand Up @@ -72,15 +73,15 @@ struct _ValveChange {
pub _message: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveSpecialConfig {
pub column: String,
pub datatype: String,
pub rule: String,
pub table: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveTableConfig {
pub table: String,
pub table_type: String,
Expand All @@ -90,7 +91,7 @@ pub struct ValveTableConfig {
pub column_order: Vec<String>,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveColumnConfig {
pub table: String,
pub column: String,
Expand All @@ -101,7 +102,7 @@ pub struct ValveColumnConfig {
pub nulltype: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveDatatypeConfig {
pub html_type: String,
pub sql_type: String,
Expand All @@ -113,7 +114,7 @@ pub struct ValveDatatypeConfig {
pub transform: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveRuleConfig {
pub description: String,
pub level: String,
Expand All @@ -124,29 +125,29 @@ pub struct ValveRuleConfig {
pub when_condition: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveTreeConstraint {
pub child: String,
pub parent: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveUnderConstraint {
pub column: String,
pub ttable: String,
pub tcolumn: String,
pub value: SerdeValue,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveForeignConstraint {
pub table: String,
pub column: String,
pub ftable: String,
pub fcolumn: String,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveConstraintConfig {
// Note that primary would be better as HashMap<String, String>, since it is not possible to
// have more than one primary key per table, but the below reflects the current implementation
Expand All @@ -159,7 +160,7 @@ pub struct ValveConstraintConfig {
pub under: HashMap<String, Vec<ValveUnderConstraint>>,
}

#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, Serialize)]
pub struct ValveConfig {
pub special: ValveSpecialConfig,
pub table: HashMap<String, ValveTableConfig>,
Expand All @@ -168,6 +169,16 @@ pub struct ValveConfig {
pub constraint: ValveConstraintConfig,
}

impl fmt::Display for ValveConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let json = serde_json::to_string(&self).expect(&format!(
"Unable to render Valve configuration: {:?} as JSON.",
self
));
write!(f, "{}", json)
}
}

/// Main entrypoint for the Valve API.
#[derive(Clone, Debug)]
pub struct Valve {
Expand Down Expand Up @@ -782,14 +793,15 @@ impl Valve {
}

/// Writes the database schema to stdout.
pub async fn dump_schema(&self) -> Result<(), ValveError> {
pub async fn dump_schema(&self) -> Result<String, ValveError> {
let setup_statements = self.get_setup_statements().await?;
let mut output = String::from("");
for table in self.get_sorted_table_list(false) {
let table_statements = setup_statements.get(table).unwrap();
let output = String::from(table_statements.join("\n"));
println!("{}\n", output);
let table_output = String::from(table_statements.join("\n"));
output.push_str(&table_output);
}
Ok(())
Ok(output)
}

/// Create all configured database tables and views if they do not already exist as configured.
Expand Down
23 changes: 16 additions & 7 deletions test/generate_random_test_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@


def get_special_tables(config):
return [k for k, v in config["special"].items() if v is not None]
return ["message", "history"] + [k for k, v in config["special"].items() if v is not None]


def get_table_columns(config, table):
Expand All @@ -35,15 +35,15 @@ def get_column_datatype(config, table, column):


def get_foreign_key(config, table, column):
return [f for f in config["constraints"]["foreign"][table] if f["column"] == column][0]
return [f for f in config["constraint"]["foreign"][table] if f["column"] == column][0]


def get_tree(config, table, column):
return [f for f in config["constraints"]["tree"][table] if f["parent"] == column][0]
return [f for f in config["constraint"]["tree"][table] if f["parent"] == column][0]


def get_under(config, table, column):
return [f for f in config["constraints"]["under"][table] if f["column"] == column][0]
return [f for f in config["constraint"]["under"][table] if f["column"] == column][0]


def get_value_from_prev_insert(config, prev_inserts, from_table, from_column, to_table, to_column):
Expand Down Expand Up @@ -172,14 +172,23 @@ def main():
sys.exit(result.returncode)
config = json.loads(result.stdout.decode())

# Get the sorted list of tables to generate:
result = subprocess.run(["./valve", "--table_order", input_table], capture_output=True)
if result.returncode != 0:
error = result.stderr.decode()
output = result.stdout.decode()
if output:
error = f"{error}\n{output}"
print(f"{error}", file=sys.stderr)
sys.exit(result.returncode)
data_tables = [t.strip() for t in result.stdout.decode().split(",")]
data_tables = [t for t in data_tables if t not in get_special_tables(config)]

# This is a record of the last inserted values for each table and column. When one column
# takes its values from another column, then we look here and fetch the last inserted value of
# the second column.
prev_inserts = {}

# The data tables to generate:
data_tables = [t for t in config["sorted_table_list"] if t not in get_special_tables(config)]

# The TSV files corresponding to each data table:
tsv_files = {}
for table in data_tables:
Expand Down

0 comments on commit 72eb8d1

Please sign in to comment.