From 6f8e09a2e79e2fb2de8e544f7436eb4cb5d89f22 Mon Sep 17 00:00:00 2001 From: Michael Cuffaro Date: Fri, 8 Dec 2023 11:27:57 -0500 Subject: [PATCH] reimplement truncate_all_tables(), add order_tables_for_deletion(), make execute_sql() private --- Makefile | 16 ++++++----- src/lib.rs | 79 +++++++++++++++++++++++------------------------------- 2 files changed, 43 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 7008e909..3e6bd336 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ MAKEFLAGS += --warn-undefined-variables build: mkdir build -.PHONY: doc readme valve_debug valve_release test sqlite_test pg_test api_test sqlite_api_test \ +.PHONY: clean doc readme valve_debug valve_release test sqlite_test pg_test api_test sqlite_api_test \ pg_qpi_test random_test_data random_test sqlite_random_test pg_random_test guess_test_data \ perf_test_data sqlite_perf_test pg_perf_test perf_test @@ -36,13 +36,13 @@ valve_debug: cargo build ln -s target/debug/ontodev_valve valve -build/valve.db: test/src/table.tsv clean valve | build - ./valve $< $@ +build/valve.db: valve test/src/table.tsv | build + ./$^ $@ test/output: mkdir -p test/output -test: sqlite_test pg_test api_test random_test +test: clean_test_db sqlite_test pg_test api_test random_test tables_to_test = column datatype rule table table1 table2 table3 table4 table5 table6 table7 table8 \ table9 table10 table11 @@ -61,6 +61,7 @@ sqlite_test: build/valve.db test/src/table.tsv | test/output pg_test: valve test/src/table.tsv | test/output @echo "Testing valve on postgresql ..." + ./$^ --drop_all postgresql:///valve_postgres ./$^ postgresql:///valve_postgres test/round_trip.sh postgresql:///valve_postgres $(word 2,$^) scripts/export.py messages postgresql:///valve_postgres $| $(tables_to_test) @@ -112,13 +113,13 @@ $(random_test_dir)/ontology: random_test_data: test/generate_random_test_data.py valve valve test/random_test_data/table.tsv | $(random_test_dir)/ontology ./$< $$(date +"%s") 100 5 $(word 3,$^) $| -sqlite_random_test: valve clean random_test_data | build test/output +sqlite_random_test: valve random_test_data | build test/output @echo "Testing with random data on sqlite ..." ./$< $(random_test_dir)/table.tsv $(sqlite_random_db) test/round_trip.sh $(sqlite_random_db) $(random_test_dir)/table.tsv @echo "Test succeeded!" -pg_random_test: valve clean random_test_data | build test/output +pg_random_test: valve random_test_data | build test/output @echo "Testing with random data on postgresql ..." ./$< $(random_test_dir)/table.tsv postgresql:///valve_postgres test/round_trip.sh postgresql:///valve_postgres $(random_test_dir)/table.tsv @@ -172,6 +173,9 @@ perf_test: sqlite_perf_test pg_perf_test clean: rm -Rf build/valve.db* build/valve_random.db* test/output $(random_test_dir)/ontology valve +clean_test_db: + rm -Rf build/valve.db + clean_guess_db: rm -Rf build/valve_guess.db diff --git a/src/lib.rs b/src/lib.rs index a4468288..1ce441a2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -177,7 +177,7 @@ impl Valve { Ok(self) } - pub async fn execute_sql(&self, sql: &str) -> Result<(), sqlx::Error> { + async fn execute_sql(&self, sql: &str) -> Result<(), sqlx::Error> { sqlx_query(&sql) .execute(self.pool.as_ref().unwrap()) .await @@ -237,28 +237,29 @@ impl Valve { Ok(self) } + pub fn order_tables_for_deletion(&self) -> Vec<&str> { + // Every other table depends on the message and history table so these will go last: + let mut sorted_tables = vec!["message", "history"]; + sorted_tables.append( + &mut self + .global_config + .get("sorted_table_list") + .and_then(|l| l.as_array()) + .and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap()))) + .and_then(|l| Some(l.collect::>())) + .unwrap(), + ); + sorted_tables.reverse(); + sorted_tables + } + /// Drop all configured tables, in reverse dependency order. /// Return an error on database problem. pub async fn drop_all_tables(&self) -> Result<&Self, sqlx::Error> { // DatabaseError // Drop all of the database tables in the reverse of their sorted order: - let sorted_tables = { - let mut sorted_tables = vec!["message", "history"]; - sorted_tables.append( - &mut self - .global_config - .get("sorted_table_list") - .and_then(|l| l.as_array()) - .and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap()))) - .and_then(|l| Some(l.collect::>())) - .unwrap(), - ); - sorted_tables.reverse(); - sorted_tables - }; - - for table in sorted_tables { + for table in self.order_tables_for_deletion() { if table != "message" && table != "history" { let sql = format!(r#"DROP VIEW IF EXISTS "{}_text_view""#, table); self.execute_sql(&sql).await?; @@ -287,39 +288,25 @@ impl Valve { /// Return an error on database problem. pub async fn truncate_all_tables(&self) -> Result<&Self, sqlx::Error> { // DatabaseError - let sorted_tables = { - let mut sorted_tables = vec!["message", "history"]; - sorted_tables.append( - &mut self - .global_config - .get("sorted_table_list") - .and_then(|l| l.as_array()) - .and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap()))) - .and_then(|l| Some(l.collect::>())) - .unwrap(), - ); - sorted_tables.reverse(); - sorted_tables + + // We must use CASCADE in the case of PostgreSQL since we cannot truncate a table, T, that + // depends on another table, T', even in the case where we have previously truncated T'. + // SQLite does not need this. However SQLite does require that the tables be truncated in + // deletion order (which means that it must be checking that T' is empty). + + let truncate_sql = |table: &str| -> String { + if self.pool.as_ref().unwrap().any_kind() == AnyKind::Postgres { + format!(r#"TRUNCATE TABLE "{}" RESTART IDENTITY CASCADE"#, table) + } else { + format!(r#"DELETE FROM "{}""#, table) + } }; - let is_postgres = self.pool.as_ref().unwrap().any_kind() == AnyKind::Postgres; - for table in sorted_tables { - let sql = format!(r#"DELETE FROM "{}""#, table); + for table in self.order_tables_for_deletion() { + let sql = truncate_sql(&table); self.execute_sql(&sql).await?; if table != "message" && table != "history" { - let sql = format!(r#"DELETE FROM "{}_conflict""#, table); - self.execute_sql(&sql).await?; - } else if table == "message" && is_postgres { - let sql = format!( - r#"ALTER SEQUENCE "{}_message_id_seq" RESTART WITH 1"#, - table - ); - self.execute_sql(&sql).await?; - } else if table == "history" && is_postgres { - let sql = format!( - r#"ALTER SEQUENCE "{}_history_id_seq" RESTART WITH 1"#, - table - ); + let sql = truncate_sql(&format!("{}_conflict", table)); self.execute_sql(&sql).await?; } }