From 7de06b8b0923328f712771587a9e0319b32ef4c7 Mon Sep 17 00:00:00 2001 From: Michael Cuffaro Date: Tue, 10 Sep 2024 13:21:29 -0400 Subject: [PATCH] when saving a table, use the column order from the db, not the current config --- src/valve.rs | 115 +++++++++++++++----------------- test/expected/messages_a1.tsv | 12 ++-- test/expected/table10.tsv | 2 +- test/random_test_data/table.tsv | 22 +++--- test/src/table.tsv | 54 +++++++-------- 5 files changed, 97 insertions(+), 108 deletions(-) diff --git a/src/valve.rs b/src/valve.rs index 4e67490b..7b46d373 100644 --- a/src/valve.rs +++ b/src/valve.rs @@ -24,7 +24,7 @@ use anyhow::Result; use csv::{QuoteStyle, ReaderBuilder, WriterBuilder}; use enquote::unquote; use futures::{executor::block_on, TryStreamExt}; -use indexmap::{IndexMap, IndexSet}; +use indexmap::IndexMap; use itertools::Itertools; use regex::Regex; use serde::{Deserialize, Serialize}; @@ -1897,18 +1897,26 @@ impl Valve { } } - /// Save all configured editable tables to their configured paths, unless save_dir is specified, - /// in which case save them there instead. + /// Save all configured savable tables to their configured paths, unless save_dir is specified, + /// in which case save them all there instead. pub async fn save_all_tables(&self, save_dir: &Option) -> Result<&Self> { let options_enabled = self.column_enabled_in_db("table", "options").await?; - // Collect tables from the 'table' table. + // Get the list of potential tables to save by querying the table table for all tables + // whose paths are TSV files: let mut tables = vec![]; let sql = { - if !options_enabled { - r#"SELECT "table" FROM "table""# + let select_from_table = String::from({ + if !options_enabled { + r#"SELECT "table" FROM "table""# + } else { + r#"SELECT "table", "options" FROM "table""# + } + }); + if self.pool.any_kind() == AnyKind::Postgres { + format!(r#"{select_from_table} WHERE "path" ILIKE '%.tsv'"#) } else { - r#"SELECT "table", "options" FROM "table""# + format!(r#"{select_from_table} WHERE LOWER("path") LIKE '%.tsv'"#) } }; let mut stream = sqlx_query(&sql).fetch(&self.pool); @@ -1931,7 +1939,7 @@ impl Valve { } } else { // If the options column is missing from the table table, then save is enabled - // by default: + // by default if the table has a .tsv path (which we have already verified above). tables.push(table.to_string()); } } @@ -1951,6 +1959,8 @@ impl Valve { if self.verbose { println!("Saving tables: {} ...", tables.join(", ")); } + // Collect the paths and possibly the options of all of the tables that were requested to be + // saved: let options_enabled = self.column_enabled_in_db("table", "options").await?; let sql = { if options_enabled { @@ -1965,7 +1975,6 @@ impl Valve { ) } }; - let mut stream = sqlx_query(&sql).fetch(&self.pool); while let Some(row) = stream.try_next().await? { let table = row @@ -1974,6 +1983,7 @@ impl Valve { .ok_or(ValveError::InputError( "No column \"table\" found in row".to_string(), ))?; + let options = row .try_get::<&str, &str>("options") .unwrap_or_default() @@ -1984,6 +1994,8 @@ impl Valve { let path = row.try_get::<&str, &str>("path").ok().unwrap_or_default(); let path = match save_dir { Some(save_dir) => { + // If the table is not saveable it can still be saved to the saved_dir if it + // has been specified and if it is not identical to the table's configured path: if !options.contains("save") { let path_dir = Path::new(path) .parent() @@ -2012,7 +2024,7 @@ impl Valve { table )) .into()); - } else if !path.ends_with(".tsv") { + } else if !path.to_lowercase().ends_with(".tsv") { return Err(ValveError::InputError(format!( "Refusing to save to non-tsv file '{}'", path @@ -2087,7 +2099,14 @@ impl Valve { } } - if self.column_enabled_in_db("table", "options").await? { + // Check if we are allowed to save the table: + if !save_path.to_lowercase().ends_with(".tsv") { + return Err(ValveError::InputError(format!( + "Refusing to save to non-tsv file '{}'", + save_path + )) + .into()); + } else if self.column_enabled_in_db("table", "options").await? { let sql = local_sql_syntax( &self.pool, &format!( @@ -2114,6 +2133,8 @@ impl Valve { }; } + // Begin by constructing a map from column names to their associated labels and formats, by + // querying the column table joined with information from the table and datatype tables. let mut columns: IndexMap = IndexMap::new(); let sql = { let mut sql_columns = r#"c."column", c."label", t."type" AS "table_type""#.to_string(); @@ -2130,11 +2151,11 @@ impl Valve { LEFT JOIN "datatype" d ON c."datatype" = d."datatype" WHERE c."table" = t."table" - AND c."table" = {SQL_PARAM}"#, + AND c."table" = {SQL_PARAM} + ORDER BY c."row_order""#, ), ) }; - let mut stream = sqlx_query(&sql).bind(table).fetch(&self.pool); while let Some(row) = stream.try_next().await? { let column = row @@ -2151,7 +2172,9 @@ impl Valve { let format = row.try_get::<&str, &str>("format").ok().unwrap_or_default(); // If the table type is not empty then it is either a special configuration table or an // internal table and, in either case, we want to ignore the label in the same way that - // we ignore it when initially reading the configuration files. + // we ignore it when initially reading the configuration files. Possibly we will want to + // allow labels for configuration table columns in the future, but for now we need to + // make sure that they are treated consistently at load-time and at save-time. if label == "" || table_type != "" { columns.insert(column.into(), (column.into(), format.into())); } else { @@ -2159,69 +2182,35 @@ impl Valve { } } - let mut labels = vec![]; - let mut formats = vec![]; - let configured_columns = &self - .config - .table - .get(table) - .ok_or(ValveError::InputError(format!( - "No table configuration found for '{}'.", - table - )))? - .column_order; - let mut leftover_columns = IndexSet::::from_iter(columns.keys().cloned()); - let mut formatted_columns = vec!["\"row_number\"".to_string()]; - - // First format the configured columns in the correct order: - for column in configured_columns { - if let Some((label, format)) = columns.get(column) { - formatted_columns.push(format!(r#""{}""#, column)); - labels.push(label); - formats.push(format); - leftover_columns.remove(column); - } - } - - // Now format any non-configured columns: - for column in &leftover_columns { - let (label, format) = columns.get(column).ok_or(ValveError::InputError(format!( - "No column '{}' found.", - column - )))?; - labels.push(label); - formats.push(format); - formatted_columns.push(format!(r#""{}""#, column)); - } - - // Construct the query to use on the basis of the formatted columns: + // Construct the query to use to retrieve the data: let query_table = format!("\"{}_text_view\"", table); let sql = format!( - r#"SELECT {} FROM {} ORDER BY "row_number""#, - formatted_columns.join(", "), + r#"SELECT "row_number", {} FROM {} ORDER BY "row_order""#, + columns + .keys() + .map(|c| format!(r#""{}""#, c)) + .collect::>() + .join(", "), query_table ); - // Combine configured and non-configured columns in preparation for writing: - let mut columns = configured_columns.clone(); - for column in &leftover_columns { - columns.push(column.to_string()); - } - + // Query the database and use the results to construct the records that will be written + // to the TSV file: let format_regex = Regex::new(PRINTF_RE)?; let mut writer = WriterBuilder::new() .delimiter(b'\t') .quote_style(QuoteStyle::Never) .from_path(save_path)?; - writer.write_record(labels)?; + let tsv_header_row = columns + .iter() + .map(|(_, (label, _))| label) + .collect::>(); + writer.write_record(tsv_header_row)?; let mut stream = sqlx_query(&sql).fetch(&self.pool); while let Some(row) = stream.try_next().await? { let mut record: Vec = vec![]; - for (i, column) in columns.iter().enumerate() { + for (column, (_, colformat)) in &columns { let cell = row.try_get::<&str, &str>(column).ok().unwrap_or_default(); - let colformat = formats.get(i).ok_or(ValveError::DataError(format!( - "Error retrieving ith format for i == {i}" - )))?; if *colformat != "" { let formatted_cell = format_cell(&colformat, &format_regex, &cell); record.push(formatted_cell.to_string()); diff --git a/test/expected/messages_a1.tsv b/test/expected/messages_a1.tsv index 02ac1f96..ad8efaec 100644 --- a/test/expected/messages_a1.tsv +++ b/test/expected/messages_a1.tsv @@ -1,10 +1,10 @@ table cell level rule message value -table E12 error option:unrecognized unrecognized option foo -table E21 warning option:overrides overrides db_table db_view -table E21 error option:reserved reserved for internal use internal -table E22 warning option:overrides overrides save db_view -table E25 warning option:overrides overrides edit no-edit -table E25 warning option:overrides overrides save no-save +table D12 error option:unrecognized unrecognized option foo +table D21 warning option:overrides overrides db_table db_view +table D21 error option:reserved reserved for internal use internal +table D22 warning option:overrides overrides save db_view +table D25 warning option:overrides overrides edit no-edit +table D25 warning option:overrides overrides save no-save table1 B5 error key:unique Values of base must be unique http://purl.obolibrary.org/obo/VO_ table1 A5 error key:primary Values of prefix must be unique VO table1 B10 error key:unique Values of base must be unique http://www.w3.org/1999/02/22-rdf-syntax-ns# diff --git a/test/expected/table10.tsv b/test/expected/table10.tsv index d17d6e5f..0ab5d757 100644 --- a/test/expected/table10.tsv +++ b/test/expected/table10.tsv @@ -1,9 +1,9 @@ foreign_column other_foreign_column numeric_foreign_column -w z b b 2 c c 3 d d 4 e e 5 +w z f f 6 g g 7 h h 8 diff --git a/test/random_test_data/table.tsv b/test/random_test_data/table.tsv index 804fc69a..fa3baf10 100644 --- a/test/random_test_data/table.tsv +++ b/test/random_test_data/table.tsv @@ -1,11 +1,11 @@ -table path description type options -column test/random_test_data/column.tsv Columns for all of the tables. column -datatype test/random_test_data/datatype.tsv Datatypes for all of the columns datatype -rule test/random_test_data/rule.tsv More complex "when" rules rule -table test/random_test_data/table.tsv All of the user-editable tables in this project. table -table1 test/random_test_data/ontology/table1.tsv The first data table -table2 test/random_test_data/ontology/table2.tsv The second data table -table3 test/random_test_data/ontology/table3.tsv The third data table -table4 test/random_test_data/ontology/table4.tsv The fourth data table -table5 test/random_test_data/ontology/table5.tsv The fifth data table -table6 test/random_test_data/ontology/table6.tsv The sixth data table (like table2 but all numeric) +table path type description options +column test/random_test_data/column.tsv column Columns for all of the tables. +datatype test/random_test_data/datatype.tsv datatype Datatypes for all of the columns +rule test/random_test_data/rule.tsv rule More complex "when" rules +table test/random_test_data/table.tsv table All of the user-editable tables in this project. +table1 test/random_test_data/ontology/table1.tsv The first data table +table2 test/random_test_data/ontology/table2.tsv The second data table +table3 test/random_test_data/ontology/table3.tsv The third data table +table4 test/random_test_data/ontology/table4.tsv The fourth data table +table5 test/random_test_data/ontology/table5.tsv The fifth data table +table6 test/random_test_data/ontology/table6.tsv The sixth data table (like table2 but all numeric) diff --git a/test/src/table.tsv b/test/src/table.tsv index 5d5b95ab..5385dd3c 100644 --- a/test/src/table.tsv +++ b/test/src/table.tsv @@ -1,27 +1,27 @@ -table path description type options -column test/src/column.tsv Columns for all of the tables. column -datatype test/src/datatype.tsv Datatypes for all of the columns datatype -rule test/src/rule.tsv More complex "when" rules rule -table test/src/table.tsv All of the user-editable tables in this project. table -table1 test/src/ontology/table1.tsv The first data table -table2 test/src/ontology/table2.tsv The second data table -table3 test/src/ontology/table3.tsv The third data table -table4 test/src/ontology/table4.tsv The fourth data table -table5 test/src/ontology/table5.tsv The fifth data table -table6 test/src/ontology/table6.tsv The sixth data table (like table2 but all numeric) -table7 test/src/ontology/table7.tsv The seventh data table -table8 test/src/ontology/table8.tsv The eightth data table foo validate_on_load conflict edit -table9 test/src/ontology/table9.tsv The ninth data table -table10 test/src/ontology/table10.tsv The tenth data table -table11 test/src/ontology/table11.tsv The eleventh data table -table12 test/src/ontology/table12.tsv The twelvth data table -table13 test/src/ontology/table13.tsv The thirteenth data table -table14 test/src/ontology/table14.tsv The fourteenth data table -table15 test/src/ontology/table15.tsv The fifteenth data table -table16 test/src/ontology/table16.tsv The sixteenth data table -view1 test/output/view1.sql db_table db_view internal -view2 test/output/view2.sh save db_view -view3 db_view -readonly1 test/output/readonly1.sh no-edit no-save no-conflict -readonly2 test/src/ontology/readonly2.tsv edit save no-edit no-save no-conflict -readonly3 test/output/readonly3.sql no-edit no-save no-conflict no-validate_on_load +table path type options description +column test/src/column.tsv column Columns for all of the tables. +datatype test/src/datatype.tsv datatype Datatypes for all of the columns +rule test/src/rule.tsv rule More complex "when" rules +table test/src/table.tsv table All of the user-editable tables in this project. +table1 test/src/ontology/table1.tsv The first data table +table2 test/src/ontology/table2.tsv The second data table +table3 test/src/ontology/table3.tsv The third data table +table4 test/src/ontology/table4.tsv The fourth data table +table5 test/src/ontology/table5.tsv The fifth data table +table6 test/src/ontology/table6.tsv The sixth data table (like table2 but all numeric) +table7 test/src/ontology/table7.tsv The seventh data table +table8 test/src/ontology/table8.tsv foo validate_on_load conflict edit The eightth data table +table9 test/src/ontology/table9.tsv The ninth data table +table10 test/src/ontology/table10.tsv The tenth data table +table11 test/src/ontology/table11.tsv The eleventh data table +table12 test/src/ontology/table12.tsv The twelvth data table +table13 test/src/ontology/table13.tsv The thirteenth data table +table14 test/src/ontology/table14.tsv The fourteenth data table +table15 test/src/ontology/table15.tsv The fifteenth data table +table16 test/src/ontology/table16.tsv The sixteenth data table +view1 test/output/view1.sql db_table db_view internal +view2 test/output/view2.sh save db_view +view3 db_view +readonly1 test/output/readonly1.sh no-edit no-save no-conflict +readonly2 test/src/ontology/readonly2.tsv edit save no-edit no-save no-conflict +readonly3 test/output/readonly3.sql no-edit no-save no-conflict no-validate_on_load