Skip to content

Commit

Permalink
implement drop_all_tables() and truncate_all_tables()
Browse files Browse the repository at this point in the history
  • Loading branch information
@lmcmicu
lmcmicu committed Dec 5, 2023
1 parent cdc99b0 commit 866e408
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 42 deletions.
6 changes: 6 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ sqlite_api_test: valve test/src/table.tsv build/valve.db test/insert_update.sh |
diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv
echo "select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id" | sqlite3 -header -tabs build/valve.db > test/output/history.tsv
diff --strip-trailing-cr -q test/expected/history.tsv test/output/history.tsv
# We drop all of the db tables because the schema for the next test (random test) is different
# from the schema used for this test.
./$< --drop_all $(word 2,$^) $(word 3,$^)
@echo "Test succeeded!"

pg_api_test: valve test/src/table.tsv test/insert_update.sh | test/output
Expand All @@ -93,6 +96,9 @@ pg_api_test: valve test/src/table.tsv test/insert_update.sh | test/output
diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv
psql postgresql:///valve_postgres -c "COPY (select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id) TO STDOUT WITH NULL AS ''" > test/output/history.tsv
tail -n +2 test/expected/history.tsv | diff --strip-trailing-cr -q test/output/history.tsv -
# We drop all of the db tables because the schema for the next test (random test) is different
# from the schema used for this test.
./$< --drop_all $(word 2,$^) postgresql:///valve_postgres
@echo "Test succeeded!"

sqlite_random_db = build/valve_random.db
Expand Down
124 changes: 86 additions & 38 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,14 @@ impl Valve {
Ok(self)
}

pub async fn execute_sql(&self, sql: &str) -> Result<(), sqlx::Error> {
sqlx_query(&sql)
.execute(self.pool.as_ref().unwrap())
.await
.expect(format!("The SQL statement: {} returned an error", sql).as_str());
Ok(())
}

/// Create all configured database tables and views
/// if they do not already exist as configured.
/// Return an error on database problems.
Expand Down Expand Up @@ -218,10 +226,7 @@ impl Valve {
for table in &tables_to_create {
let table_statements = setup_statements.get(table).unwrap();
for stmt in table_statements {
sqlx_query(stmt)
.execute(pool)
.await
.expect(format!("The SQL statement: {} returned an error", stmt).as_str());
self.execute_sql(stmt).await?;
}
if self.verbose {
let output = String::from(table_statements.join("\n"));
Expand All @@ -234,10 +239,38 @@ impl Valve {

/// Drop all configured tables, in reverse dependency order.
/// Return an error on database problem.
pub fn drop_all_tables(&self) -> Result<&Self, sqlx::Error> {
pub async fn drop_all_tables(&self) -> Result<&Self, sqlx::Error> {
// DatabaseError

// TODO NEXT
// Drop all of the database tables in the reverse of their sorted order:
let sorted_tables = {
let mut sorted_tables = vec!["message", "history"];
sorted_tables.append(
&mut self
.global_config
.get("sorted_table_list")
.and_then(|l| l.as_array())
.and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap())))
.and_then(|l| Some(l.collect::<Vec<_>>()))
.unwrap(),
);
sorted_tables.reverse();
sorted_tables
};

for table in sorted_tables {
if table != "message" && table != "history" {
let sql = format!(r#"DROP VIEW IF EXISTS "{}_text_view""#, table);
self.execute_sql(&sql).await?;
let sql = format!(r#"DROP VIEW IF EXISTS "{}_view""#, table);
self.execute_sql(&sql).await?;
let sql = format!(r#"DROP TABLE IF EXISTS "{}_conflict""#, table);
self.execute_sql(&sql).await?;
}
let sql = format!(r#"DROP TABLE IF EXISTS "{}""#, table);
self.execute_sql(&sql).await?;
}

Ok(self)
}

Expand All @@ -252,9 +285,45 @@ impl Valve {

/// Truncate all configured tables, in reverse dependency order.
/// Return an error on database problem.
pub fn truncate_all_tables(&self) -> Result<&Self, sqlx::Error> {
pub async fn truncate_all_tables(&self) -> Result<&Self, sqlx::Error> {
// DatabaseError
// TODO
let sorted_tables = {
let mut sorted_tables = vec!["message", "history"];
sorted_tables.append(
&mut self
.global_config
.get("sorted_table_list")
.and_then(|l| l.as_array())
.and_then(|l| Some(l.iter().map(|i| i.as_str().unwrap())))
.and_then(|l| Some(l.collect::<Vec<_>>()))
.unwrap(),
);
sorted_tables.reverse();
sorted_tables
};

let is_postgres = self.pool.as_ref().unwrap().any_kind() == AnyKind::Postgres;
for table in sorted_tables {
let sql = format!(r#"DELETE FROM "{}""#, table);
self.execute_sql(&sql).await?;
if table != "message" && table != "history" {
let sql = format!(r#"DELETE FROM "{}_conflict""#, table);
self.execute_sql(&sql).await?;
} else if table == "message" && is_postgres {
let sql = format!(
r#"ALTER SEQUENCE "{}_message_id_seq" RESTART WITH 1"#,
table
);
self.execute_sql(&sql).await?;
} else if table == "history" && is_postgres {
let sql = format!(
r#"ALTER SEQUENCE "{}_history_id_seq" RESTART WITH 1"#,
table
);
self.execute_sql(&sql).await?;
}
}

Ok(self)
}

Expand All @@ -276,7 +345,8 @@ impl Valve {
// DatabaseError

self.create_missing_tables().await?;
//self.truncate_all_tables();
self.truncate_all_tables().await?;

if let Some(pool) = &self.pool {
if pool.any_kind() == AnyKind::Sqlite {
sqlx_query("PRAGMA foreign_keys = ON").execute(pool).await?;
Expand Down Expand Up @@ -1232,6 +1302,8 @@ pub fn get_parsed_structure_conditions(
parsed_structure_conditions
}

// TODO: Modify this function so that it no longer returns the DROP statement, once you have
// removed the old valve functions that require it.
/// Given the name of a table and a database connection pool, generate SQL for creating a view
/// based on the table that provides a unified representation of the normal and conflict versions
/// of the table, plus columns summarising the information associated with the given table that is
Expand Down Expand Up @@ -1352,6 +1424,8 @@ fn get_sql_for_standard_view(table: &str, pool: &AnyPool) -> (String, String) {
(drop_view_sql, create_view_sql)
}

// TODO: Modify this function so that it no longer returns the DROP statement, once you have
// removed the old valve functions that require it.
/// Given the tables configuration map, the name of a table and a database connection pool,
/// generate SQL for creating a more user-friendly version of the view than the one generated by
/// [get_sql_for_standard_view()]. Unlike the standard view generated by that function, the view
Expand Down Expand Up @@ -1484,11 +1558,8 @@ pub async fn get_setup_statements(
table_statements.append(&mut statements);
}

let (drop_view_sql, create_view_sql) = get_sql_for_standard_view(&table_name, pool);
let (drop_text_view_sql, create_text_view_sql) =
get_sql_for_text_view(tables_config, &table_name, pool);
table_statements.push(drop_text_view_sql);
table_statements.push(drop_view_sql);
let (_, create_view_sql) = get_sql_for_standard_view(&table_name, pool);
let (_, create_text_view_sql) = get_sql_for_text_view(tables_config, &table_name, pool);
table_statements.push(create_view_sql);
table_statements.push(create_text_view_sql);

Expand All @@ -1497,14 +1568,6 @@ pub async fn get_setup_statements(

// Generate DDL for the history table:
let mut history_statements = vec![];
history_statements.push({
let mut sql = r#"DROP TABLE IF EXISTS "history""#.to_string();
if pool.any_kind() == AnyKind::Postgres {
sql.push_str(" CASCADE");
}
sql.push_str(";");
sql
});
history_statements.push(format!(
indoc! {r#"
CREATE TABLE IF NOT EXISTS "history" (
Expand Down Expand Up @@ -1541,14 +1604,6 @@ pub async fn get_setup_statements(

// Generate DDL for the message table:
let mut message_statements = vec![];
message_statements.push({
let mut sql = r#"DROP TABLE IF EXISTS "message""#.to_string();
if pool.any_kind() == AnyKind::Postgres {
sql.push_str(" CASCADE");
}
sql.push_str(";");
sql
});
message_statements.push(format!(
indoc! {r#"
CREATE TABLE IF NOT EXISTS "message" (
Expand Down Expand Up @@ -4660,14 +4715,7 @@ fn get_table_ddl(
table_name: &String,
pool: &AnyPool,
) -> Vec<String> {
// TODO: Don't generate "drop" statements in this function. It will be done elsewhere. This
// function should only generate creation statements.
let mut drop_table_sql = format!(r#"DROP TABLE IF EXISTS "{}""#, table_name);
if pool.any_kind() == AnyKind::Postgres {
drop_table_sql.push_str(" CASCADE");
}
drop_table_sql.push_str(";");
let mut statements = vec![drop_table_sql];
let mut statements = vec![];
let mut create_lines = vec![
format!(r#"CREATE TABLE IF NOT EXISTS "{}" ("#, table_name),
String::from(r#" "row_number" BIGINT,"#),
Expand Down
18 changes: 14 additions & 4 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ use argparse::{ArgumentParser, Store, StoreTrue};

use ontodev_valve::{
get_compiled_datatype_conditions, get_compiled_rule_conditions,
get_parsed_structure_conditions, valve, valve_grammar::StartParser, ValveCommand,
Valve
get_parsed_structure_conditions, valve, valve_grammar::StartParser, Valve, ValveCommand,
};
use serde_json::{from_str, Value as SerdeValue};
use std::{env, process};
Expand All @@ -20,6 +19,7 @@ fn cli_args_valid(source: &str, destination: &str, dump_config: bool) -> bool {
async fn main() -> Result<(), sqlx::Error> {
let mut api_test = false;
let mut dump_config = false;
let mut drop_all = false;
let mut create_only = false;
let mut config_table = String::new();
let mut verbose = false;
Expand Down Expand Up @@ -49,6 +49,11 @@ async fn main() -> Result<(), sqlx::Error> {
r#"Read the configuration referred to by SOURCE and send it to stdout as a
JSON-formatted string."#,
);
ap.refer(&mut drop_all).add_option(
&["--drop_all"],
StoreTrue,
r#"Drop all tables in the database."#,
);
ap.refer(&mut create_only).add_option(
&["--create_only"],
StoreTrue,
Expand Down Expand Up @@ -146,6 +151,10 @@ async fn main() -> Result<(), sqlx::Error> {

let config = serde_json::to_string(config).unwrap();
println!("{}", config);
} else if drop_all {
let valve =
Valve::build(&source, &config_table, &destination, verbose, initial_load).await?;
valve.drop_all_tables().await?;
} else if create_only {
valve(
&source,
Expand All @@ -157,9 +166,10 @@ async fn main() -> Result<(), sqlx::Error> {
)
.await?;
} else {
let mut valve = Valve::build(&source, &config_table, &destination, verbose,
initial_load).await?;
let mut valve =
Valve::build(&source, &config_table, &destination, verbose, initial_load).await?;
valve.load_all_tables(true).await?;

// valve(
// &source,
// &destination,
Expand Down

0 comments on commit 866e408

Please sign in to comment.