Skip to content

Commit

Permalink
when saving a table, use the column order from the db, not the curren…
Browse files Browse the repository at this point in the history 
…t config
  • Loading branch information
@lmcmicu
lmcmicu committed Sep 10, 2024
1 parent c02ca64 commit 7de06b8
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 108 deletions.
115 changes: 52 additions & 63 deletions src/valve.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use anyhow::Result;
use csv::{QuoteStyle, ReaderBuilder, WriterBuilder};
use enquote::unquote;
use futures::{executor::block_on, TryStreamExt};
use indexmap::{IndexMap, IndexSet};
use indexmap::IndexMap;
use itertools::Itertools;
use regex::Regex;
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -1897,18 +1897,26 @@ impl Valve {
}
}

/// Save all configured editable tables to their configured paths, unless save_dir is specified,
/// in which case save them there instead.
/// Save all configured savable tables to their configured paths, unless save_dir is specified,
/// in which case save them all there instead.
pub async fn save_all_tables(&self, save_dir: &Option<String>) -> Result<&Self> {
let options_enabled = self.column_enabled_in_db("table", "options").await?;

// Collect tables from the 'table' table.
// Get the list of potential tables to save by querying the table table for all tables
// whose paths are TSV files:
let mut tables = vec![];
let sql = {
if !options_enabled {
r#"SELECT "table" FROM "table""#
let select_from_table = String::from({
if !options_enabled {
r#"SELECT "table" FROM "table""#
} else {
r#"SELECT "table", "options" FROM "table""#
}
});
if self.pool.any_kind() == AnyKind::Postgres {
format!(r#"{select_from_table} WHERE "path" ILIKE '%.tsv'"#)
} else {
r#"SELECT "table", "options" FROM "table""#
format!(r#"{select_from_table} WHERE LOWER("path") LIKE '%.tsv'"#)
}
};
let mut stream = sqlx_query(&sql).fetch(&self.pool);
Expand All @@ -1931,7 +1939,7 @@ impl Valve {
}
} else {
// If the options column is missing from the table table, then save is enabled
// by default:
// by default if the table has a .tsv path (which we have already verified above).
tables.push(table.to_string());
}
}
Expand All @@ -1951,6 +1959,8 @@ impl Valve {
if self.verbose {
println!("Saving tables: {} ...", tables.join(", "));
}
// Collect the paths and possibly the options of all of the tables that were requested to be
// saved:
let options_enabled = self.column_enabled_in_db("table", "options").await?;
let sql = {
if options_enabled {
Expand All @@ -1965,7 +1975,6 @@ impl Valve {
)
}
};

let mut stream = sqlx_query(&sql).fetch(&self.pool);
while let Some(row) = stream.try_next().await? {
let table = row
Expand All @@ -1974,6 +1983,7 @@ impl Valve {
.ok_or(ValveError::InputError(
"No column \"table\" found in row".to_string(),
))?;

let options = row
.try_get::<&str, &str>("options")
.unwrap_or_default()
Expand All @@ -1984,6 +1994,8 @@ impl Valve {
let path = row.try_get::<&str, &str>("path").ok().unwrap_or_default();
let path = match save_dir {
Some(save_dir) => {
// If the table is not saveable it can still be saved to the saved_dir if it
// has been specified and if it is not identical to the table's configured path:
if !options.contains("save") {
let path_dir = Path::new(path)
.parent()
Expand Down Expand Up @@ -2012,7 +2024,7 @@ impl Valve {
table
))
.into());
} else if !path.ends_with(".tsv") {
} else if !path.to_lowercase().ends_with(".tsv") {
return Err(ValveError::InputError(format!(
"Refusing to save to non-tsv file '{}'",
path
Expand Down Expand Up @@ -2087,7 +2099,14 @@ impl Valve {
}
}

if self.column_enabled_in_db("table", "options").await? {
// Check if we are allowed to save the table:
if !save_path.to_lowercase().ends_with(".tsv") {
return Err(ValveError::InputError(format!(
"Refusing to save to non-tsv file '{}'",
save_path
))
.into());
} else if self.column_enabled_in_db("table", "options").await? {
let sql = local_sql_syntax(
&self.pool,
&format!(
Expand All @@ -2114,6 +2133,8 @@ impl Valve {
};
}

// Begin by constructing a map from column names to their associated labels and formats, by
// querying the column table joined with information from the table and datatype tables.
let mut columns: IndexMap<String, (String, String)> = IndexMap::new();
let sql = {
let mut sql_columns = r#"c."column", c."label", t."type" AS "table_type""#.to_string();
Expand All @@ -2130,11 +2151,11 @@ impl Valve {
LEFT JOIN "datatype" d
ON c."datatype" = d."datatype"
WHERE c."table" = t."table"
AND c."table" = {SQL_PARAM}"#,
AND c."table" = {SQL_PARAM}
ORDER BY c."row_order""#,
),
)
};

let mut stream = sqlx_query(&sql).bind(table).fetch(&self.pool);
while let Some(row) = stream.try_next().await? {
let column = row
Expand All @@ -2151,77 +2172,45 @@ impl Valve {
let format = row.try_get::<&str, &str>("format").ok().unwrap_or_default();
// If the table type is not empty then it is either a special configuration table or an
// internal table and, in either case, we want to ignore the label in the same way that
// we ignore it when initially reading the configuration files.
// we ignore it when initially reading the configuration files. Possibly we will want to
// allow labels for configuration table columns in the future, but for now we need to
// make sure that they are treated consistently at load-time and at save-time.
if label == "" || table_type != "" {
columns.insert(column.into(), (column.into(), format.into()));
} else {
columns.insert(column.into(), (label.into(), format.into()));
}
}

let mut labels = vec![];
let mut formats = vec![];
let configured_columns = &self
.config
.table
.get(table)
.ok_or(ValveError::InputError(format!(
"No table configuration found for '{}'.",
table
)))?
.column_order;
let mut leftover_columns = IndexSet::<String>::from_iter(columns.keys().cloned());
let mut formatted_columns = vec!["\"row_number\"".to_string()];

// First format the configured columns in the correct order:
for column in configured_columns {
if let Some((label, format)) = columns.get(column) {
formatted_columns.push(format!(r#""{}""#, column));
labels.push(label);
formats.push(format);
leftover_columns.remove(column);
}
}

// Now format any non-configured columns:
for column in &leftover_columns {
let (label, format) = columns.get(column).ok_or(ValveError::InputError(format!(
"No column '{}' found.",
column
)))?;
labels.push(label);
formats.push(format);
formatted_columns.push(format!(r#""{}""#, column));
}

// Construct the query to use on the basis of the formatted columns:
// Construct the query to use to retrieve the data:
let query_table = format!("\"{}_text_view\"", table);
let sql = format!(
r#"SELECT {} FROM {} ORDER BY "row_number""#,
formatted_columns.join(", "),
r#"SELECT "row_number", {} FROM {} ORDER BY "row_order""#,
columns
.keys()
.map(|c| format!(r#""{}""#, c))
.collect::<Vec<_>>()
.join(", "),
query_table
);

// Combine configured and non-configured columns in preparation for writing:
let mut columns = configured_columns.clone();
for column in &leftover_columns {
columns.push(column.to_string());
}

// Query the database and use the results to construct the records that will be written
// to the TSV file:
let format_regex = Regex::new(PRINTF_RE)?;
let mut writer = WriterBuilder::new()
.delimiter(b'\t')
.quote_style(QuoteStyle::Never)
.from_path(save_path)?;
writer.write_record(labels)?;
let tsv_header_row = columns
.iter()
.map(|(_, (label, _))| label)
.collect::<Vec<_>>();
writer.write_record(tsv_header_row)?;
let mut stream = sqlx_query(&sql).fetch(&self.pool);
while let Some(row) = stream.try_next().await? {
let mut record: Vec<String> = vec![];
for (i, column) in columns.iter().enumerate() {
for (column, (_, colformat)) in &columns {
let cell = row.try_get::<&str, &str>(column).ok().unwrap_or_default();
let colformat = formats.get(i).ok_or(ValveError::DataError(format!(
"Error retrieving ith format for i == {i}"
)))?;
if *colformat != "" {
let formatted_cell = format_cell(&colformat, &format_regex, &cell);
record.push(formatted_cell.to_string());
Expand Down
12 changes: 6 additions & 6 deletions test/expected/messages_a1.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
table cell level rule message value
table E12 error option:unrecognized unrecognized option foo
table E21 warning option:overrides overrides db_table db_view
table E21 error option:reserved reserved for internal use internal
table E22 warning option:overrides overrides save db_view
table E25 warning option:overrides overrides edit no-edit
table E25 warning option:overrides overrides save no-save
table D12 error option:unrecognized unrecognized option foo
table D21 warning option:overrides overrides db_table db_view
table D21 error option:reserved reserved for internal use internal
table D22 warning option:overrides overrides save db_view
table D25 warning option:overrides overrides edit no-edit
table D25 warning option:overrides overrides save no-save
table1 B5 error key:unique Values of base must be unique http://purl.obolibrary.org/obo/VO_
table1 A5 error key:primary Values of prefix must be unique VO
table1 B10 error key:unique Values of base must be unique http://www.w3.org/1999/02/22-rdf-syntax-ns#
Expand Down
2 changes: 1 addition & 1 deletion test/expected/table10.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
foreign_column other_foreign_column numeric_foreign_column
w z
b b 2
c c 3
d d 4
e e 5
w z
f f 6
g g 7
h h 8
Expand Down
22 changes: 11 additions & 11 deletions test/random_test_data/table.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
table path description type options
column test/random_test_data/column.tsv Columns for all of the tables. column
datatype test/random_test_data/datatype.tsv Datatypes for all of the columns datatype
rule test/random_test_data/rule.tsv More complex "when" rules rule
table test/random_test_data/table.tsv All of the user-editable tables in this project. table
table1 test/random_test_data/ontology/table1.tsv The first data table
table2 test/random_test_data/ontology/table2.tsv The second data table
table3 test/random_test_data/ontology/table3.tsv The third data table
table4 test/random_test_data/ontology/table4.tsv The fourth data table
table5 test/random_test_data/ontology/table5.tsv The fifth data table
table6 test/random_test_data/ontology/table6.tsv The sixth data table (like table2 but all numeric)
table path type description options
column test/random_test_data/column.tsv column Columns for all of the tables.
datatype test/random_test_data/datatype.tsv datatype Datatypes for all of the columns
rule test/random_test_data/rule.tsv rule More complex "when" rules
table test/random_test_data/table.tsv table All of the user-editable tables in this project.
table1 test/random_test_data/ontology/table1.tsv The first data table
table2 test/random_test_data/ontology/table2.tsv The second data table
table3 test/random_test_data/ontology/table3.tsv The third data table
table4 test/random_test_data/ontology/table4.tsv The fourth data table
table5 test/random_test_data/ontology/table5.tsv The fifth data table
table6 test/random_test_data/ontology/table6.tsv The sixth data table (like table2 but all numeric)
54 changes: 27 additions & 27 deletions test/src/table.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
table path description type options
column test/src/column.tsv Columns for all of the tables. column
datatype test/src/datatype.tsv Datatypes for all of the columns datatype
rule test/src/rule.tsv More complex "when" rules rule
table test/src/table.tsv All of the user-editable tables in this project. table
table1 test/src/ontology/table1.tsv The first data table
table2 test/src/ontology/table2.tsv The second data table
table3 test/src/ontology/table3.tsv The third data table
table4 test/src/ontology/table4.tsv The fourth data table
table5 test/src/ontology/table5.tsv The fifth data table
table6 test/src/ontology/table6.tsv The sixth data table (like table2 but all numeric)
table7 test/src/ontology/table7.tsv The seventh data table
table8 test/src/ontology/table8.tsv The eightth data table foo validate_on_load conflict edit
table9 test/src/ontology/table9.tsv The ninth data table
table10 test/src/ontology/table10.tsv The tenth data table
table11 test/src/ontology/table11.tsv The eleventh data table
table12 test/src/ontology/table12.tsv The twelvth data table
table13 test/src/ontology/table13.tsv The thirteenth data table
table14 test/src/ontology/table14.tsv The fourteenth data table
table15 test/src/ontology/table15.tsv The fifteenth data table
table16 test/src/ontology/table16.tsv The sixteenth data table
view1 test/output/view1.sql db_table db_view internal
view2 test/output/view2.sh save db_view
view3 db_view
readonly1 test/output/readonly1.sh no-edit no-save no-conflict
readonly2 test/src/ontology/readonly2.tsv edit save no-edit no-save no-conflict
readonly3 test/output/readonly3.sql no-edit no-save no-conflict no-validate_on_load
table path type options description
column test/src/column.tsv column Columns for all of the tables.
datatype test/src/datatype.tsv datatype Datatypes for all of the columns
rule test/src/rule.tsv rule More complex "when" rules
table test/src/table.tsv table All of the user-editable tables in this project.
table1 test/src/ontology/table1.tsv The first data table
table2 test/src/ontology/table2.tsv The second data table
table3 test/src/ontology/table3.tsv The third data table
table4 test/src/ontology/table4.tsv The fourth data table
table5 test/src/ontology/table5.tsv The fifth data table
table6 test/src/ontology/table6.tsv The sixth data table (like table2 but all numeric)
table7 test/src/ontology/table7.tsv The seventh data table
table8 test/src/ontology/table8.tsv foo validate_on_load conflict edit The eightth data table
table9 test/src/ontology/table9.tsv The ninth data table
table10 test/src/ontology/table10.tsv The tenth data table
table11 test/src/ontology/table11.tsv The eleventh data table
table12 test/src/ontology/table12.tsv The twelvth data table
table13 test/src/ontology/table13.tsv The thirteenth data table
table14 test/src/ontology/table14.tsv The fourteenth data table
table15 test/src/ontology/table15.tsv The fifteenth data table
table16 test/src/ontology/table16.tsv The sixteenth data table
view1 test/output/view1.sql db_table db_view internal
view2 test/output/view2.sh save db_view
view3 db_view
readonly1 test/output/readonly1.sh no-edit no-save no-conflict
readonly2 test/src/ontology/readonly2.tsv edit save no-edit no-save no-conflict
readonly3 test/output/readonly3.sql no-edit no-save no-conflict no-validate_on_load

0 comments on commit 7de06b8

Please sign in to comment.