From 866e408f9b8c398bce3e750ddb17b71c846621bc Mon Sep 17 00:00:00 2001 From: Michael Cuffaro Date: Tue, 5 Dec 2023 14:02:08 -0500 Subject: [PATCH] implement drop_all_tables() and truncate_all_tables() --- Makefile | 6 +++ src/lib.rs | 124 ++++++++++++++++++++++++++++++++++++---------------- src/main.rs | 18 ++++++-- 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index 5832e279..7008e909 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ sqlite_api_test: valve test/src/table.tsv build/valve.db test/insert_update.sh | diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv echo "select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id" | sqlite3 -header -tabs build/valve.db > test/output/history.tsv diff --strip-trailing-cr -q test/expected/history.tsv test/output/history.tsv + # We drop all of the db tables because the schema for the next test (random test) is different + # from the schema used for this test. + ./$< --drop_all $(word 2,$^) $(word 3,$^) @echo "Test succeeded!" pg_api_test: valve test/src/table.tsv test/insert_update.sh | test/output @@ -93,6 +96,9 @@ pg_api_test: valve test/src/table.tsv test/insert_update.sh | test/output diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv psql postgresql:///valve_postgres -c "COPY (select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id) TO STDOUT WITH NULL AS ''" > test/output/history.tsv tail -n +2 test/expected/history.tsv | diff --strip-trailing-cr -q test/output/history.tsv - + # We drop all of the db tables because the schema for the next test (random test) is different + # from the schema used for this test. + ./$< --drop_all $(word 2,$^) postgresql:///valve_postgres @echo "Test succeeded!" sqlite_random_db = build/valve_random.db diff --git a/src/lib.rs b/src/lib.rs index 92b43156..a4468288 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -177,6 +177,14 @@ impl Valve { Ok(self) } + pub async fn execute_sql(&self, sql: &str) -> Result<(), sqlx::Error> { + sqlx_query(&sql) + .execute(self.pool.as_ref().unwrap()) + .await + .expect(format!("The SQL statement: {} returned an error", sql).as_str()); + Ok(()) + } + /// Create all configured database tables and views /// if they do not already exist as configured. /// Return an error on database problems. @@ -218,10 +226,7 @@ impl Valve { for table in &tables_to_create { let table_statements = setup_statements.get(table).unwrap(); for stmt in table_statements { - sqlx_query(stmt) - .execute(pool) - .await - .expect(format!("The SQL statement: {} returned an error", stmt).as_str()); + self.execute_sql(stmt).await?; } if self.verbose { let output = String::from(table_statements.join("\n")); @@ -234,10 +239,38 @@ impl Valve { /// Drop all configured tables, in reverse dependency order. /// Return an error on database problem. - pub fn drop_all_tables(&self) -> Result<&Self, sqlx::Error> { + pub async fn drop_all_tables(&self) -> Result<&Self, sqlx::Error> { // DatabaseError - // TODO NEXT + // Drop all of the database tables in the reverse of their sorted order: + let sorted_tables = { + let mut sorted_tables = vec!["message", "history"]; + sorted_tables.append( + &mut self + .global_config + .get("sorted_table_list") + .and_then(|l| l.as_array()) + .and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap()))) + .and_then(|l| Some(l.collect::>())) + .unwrap(), + ); + sorted_tables.reverse(); + sorted_tables + }; + + for table in sorted_tables { + if table != "message" && table != "history" { + let sql = format!(r#"DROP VIEW IF EXISTS "{}_text_view""#, table); + self.execute_sql(&sql).await?; + let sql = format!(r#"DROP VIEW IF EXISTS "{}_view""#, table); + self.execute_sql(&sql).await?; + let sql = format!(r#"DROP TABLE IF EXISTS "{}_conflict""#, table); + self.execute_sql(&sql).await?; + } + let sql = format!(r#"DROP TABLE IF EXISTS "{}""#, table); + self.execute_sql(&sql).await?; + } + Ok(self) } @@ -252,9 +285,45 @@ impl Valve { /// Truncate all configured tables, in reverse dependency order. /// Return an error on database problem. - pub fn truncate_all_tables(&self) -> Result<&Self, sqlx::Error> { + pub async fn truncate_all_tables(&self) -> Result<&Self, sqlx::Error> { // DatabaseError - // TODO + let sorted_tables = { + let mut sorted_tables = vec!["message", "history"]; + sorted_tables.append( + &mut self + .global_config + .get("sorted_table_list") + .and_then(|l| l.as_array()) + .and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap()))) + .and_then(|l| Some(l.collect::>())) + .unwrap(), + ); + sorted_tables.reverse(); + sorted_tables + }; + + let is_postgres = self.pool.as_ref().unwrap().any_kind() == AnyKind::Postgres; + for table in sorted_tables { + let sql = format!(r#"DELETE FROM "{}""#, table); + self.execute_sql(&sql).await?; + if table != "message" && table != "history" { + let sql = format!(r#"DELETE FROM "{}_conflict""#, table); + self.execute_sql(&sql).await?; + } else if table == "message" && is_postgres { + let sql = format!( + r#"ALTER SEQUENCE "{}_message_id_seq" RESTART WITH 1"#, + table + ); + self.execute_sql(&sql).await?; + } else if table == "history" && is_postgres { + let sql = format!( + r#"ALTER SEQUENCE "{}_history_id_seq" RESTART WITH 1"#, + table + ); + self.execute_sql(&sql).await?; + } + } + Ok(self) } @@ -276,7 +345,8 @@ impl Valve { // DatabaseError self.create_missing_tables().await?; - //self.truncate_all_tables(); + self.truncate_all_tables().await?; + if let Some(pool) = &self.pool { if pool.any_kind() == AnyKind::Sqlite { sqlx_query("PRAGMA foreign_keys = ON").execute(pool).await?; @@ -1232,6 +1302,8 @@ pub fn get_parsed_structure_conditions( parsed_structure_conditions } +// TODO: Modify this function so that it no longer returns the DROP statement, once you have +// removed the old valve functions that require it. /// Given the name of a table and a database connection pool, generate SQL for creating a view /// based on the table that provides a unified representation of the normal and conflict versions /// of the table, plus columns summarising the information associated with the given table that is @@ -1352,6 +1424,8 @@ fn get_sql_for_standard_view(table: &str, pool: &AnyPool) -> (String, String) { (drop_view_sql, create_view_sql) } +// TODO: Modify this function so that it no longer returns the DROP statement, once you have +// removed the old valve functions that require it. /// Given the tables configuration map, the name of a table and a database connection pool, /// generate SQL for creating a more user-friendly version of the view than the one generated by /// [get_sql_for_standard_view()]. Unlike the standard view generated by that function, the view @@ -1484,11 +1558,8 @@ pub async fn get_setup_statements( table_statements.append(&mut statements); } - let (drop_view_sql, create_view_sql) = get_sql_for_standard_view(&table_name, pool); - let (drop_text_view_sql, create_text_view_sql) = - get_sql_for_text_view(tables_config, &table_name, pool); - table_statements.push(drop_text_view_sql); - table_statements.push(drop_view_sql); + let (_, create_view_sql) = get_sql_for_standard_view(&table_name, pool); + let (_, create_text_view_sql) = get_sql_for_text_view(tables_config, &table_name, pool); table_statements.push(create_view_sql); table_statements.push(create_text_view_sql); @@ -1497,14 +1568,6 @@ pub async fn get_setup_statements( // Generate DDL for the history table: let mut history_statements = vec![]; - history_statements.push({ - let mut sql = r#"DROP TABLE IF EXISTS "history""#.to_string(); - if pool.any_kind() == AnyKind::Postgres { - sql.push_str(" CASCADE"); - } - sql.push_str(";"); - sql - }); history_statements.push(format!( indoc! {r#" CREATE TABLE IF NOT EXISTS "history" ( @@ -1541,14 +1604,6 @@ pub async fn get_setup_statements( // Generate DDL for the message table: let mut message_statements = vec![]; - message_statements.push({ - let mut sql = r#"DROP TABLE IF EXISTS "message""#.to_string(); - if pool.any_kind() == AnyKind::Postgres { - sql.push_str(" CASCADE"); - } - sql.push_str(";"); - sql - }); message_statements.push(format!( indoc! {r#" CREATE TABLE IF NOT EXISTS "message" ( @@ -4660,14 +4715,7 @@ fn get_table_ddl( table_name: &String, pool: &AnyPool, ) -> Vec { - // TODO: Don't generate "drop" statements in this function. It will be done elsewhere. This - // function should only generate creation statements. - let mut drop_table_sql = format!(r#"DROP TABLE IF EXISTS "{}""#, table_name); - if pool.any_kind() == AnyKind::Postgres { - drop_table_sql.push_str(" CASCADE"); - } - drop_table_sql.push_str(";"); - let mut statements = vec![drop_table_sql]; + let mut statements = vec![]; let mut create_lines = vec![ format!(r#"CREATE TABLE IF NOT EXISTS "{}" ("#, table_name), String::from(r#" "row_number" BIGINT,"#), diff --git a/src/main.rs b/src/main.rs index a48a88b2..408e29bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,8 +6,7 @@ use argparse::{ArgumentParser, Store, StoreTrue}; use ontodev_valve::{ get_compiled_datatype_conditions, get_compiled_rule_conditions, - get_parsed_structure_conditions, valve, valve_grammar::StartParser, ValveCommand, - Valve + get_parsed_structure_conditions, valve, valve_grammar::StartParser, Valve, ValveCommand, }; use serde_json::{from_str, Value as SerdeValue}; use std::{env, process}; @@ -20,6 +19,7 @@ fn cli_args_valid(source: &str, destination: &str, dump_config: bool) -> bool { async fn main() -> Result<(), sqlx::Error> { let mut api_test = false; let mut dump_config = false; + let mut drop_all = false; let mut create_only = false; let mut config_table = String::new(); let mut verbose = false; @@ -49,6 +49,11 @@ async fn main() -> Result<(), sqlx::Error> { r#"Read the configuration referred to by SOURCE and send it to stdout as a JSON-formatted string."#, ); + ap.refer(&mut drop_all).add_option( + &["--drop_all"], + StoreTrue, + r#"Drop all tables in the database."#, + ); ap.refer(&mut create_only).add_option( &["--create_only"], StoreTrue, @@ -146,6 +151,10 @@ async fn main() -> Result<(), sqlx::Error> { let config = serde_json::to_string(config).unwrap(); println!("{}", config); + } else if drop_all { + let valve = + Valve::build(&source, &config_table, &destination, verbose, initial_load).await?; + valve.drop_all_tables().await?; } else if create_only { valve( &source, @@ -157,9 +166,10 @@ async fn main() -> Result<(), sqlx::Error> { ) .await?; } else { - let mut valve = Valve::build(&source, &config_table, &destination, verbose, - initial_load).await?; + let mut valve = + Valve::build(&source, &config_table, &destination, verbose, initial_load).await?; valve.load_all_tables(true).await?; + // valve( // &source, // &destination,