Merge pull request #102 from ontodev/use-list-subcondition

Use subcondition of list in get_matching_values

.github/workflows/valve-tests.yml

@@ -43,4 +43,4 @@ jobs:
           pip3 install -r requirements.txt
       - name: Run tests on both sqlite and on postgresql
         run: |
-          make test penguin_test
+          make test penguin_test perf_test

Makefile

@@ -99,7 +99,7 @@ sqlite_api_test: valve test/src/table.tsv build/valve.db test/insert_update.sh |
 	$(word 4,$^) $(word 3,$^) $(word 2,$^)
 	scripts/export_messages.py $(word 3,$^) $| $(tables_to_test)
 	diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv
-	echo "select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id" | sqlite3 -header -tabs build/valve.db > test/output/history.tsv
+	echo "select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 16 order by history_id" | sqlite3 -header -tabs build/valve.db > test/output/history.tsv
 	diff --strip-trailing-cr -q test/expected/history.tsv test/output/history.tsv
 	# We drop all of the db tables because the schema for the next test (random test) is different
 	# from the schema used for this test.
@@ -114,7 +114,7 @@ pg_api_test: valve test/src/table.tsv test/insert_update.sh | test/output
 	$(word 3,$^) $(pg_connect_string) $(word 2,$^)
 	scripts/export_messages.py $(pg_connect_string) $| $(tables_to_test)
 	diff --strip-trailing-cr -q test/expected/messages_after_api_test.tsv test/output/messages.tsv
-	psql $(pg_connect_string) -c "COPY (select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 15 order by history_id) TO STDOUT WITH NULL AS ''" > test/output/history.tsv
+	psql $(pg_connect_string) -c "COPY (select \"history_id\", \"table\", \"row\", \"from\", \"to\", \"summary\", \"user\", \"undone_by\" from history where history_id < 16 order by history_id) TO STDOUT WITH NULL AS ''" > test/output/history.tsv
 	tail -n +2 test/expected/history.tsv | diff --strip-trailing-cr -q test/output/history.tsv -
 	# We drop all of the db tables because the schema for the next test (random test) is different
 	# from the schema used for this test.
@@ -151,7 +151,12 @@ pg_random_test: valve random_test_data | build test/output
 test/penguins/src/data:
 	mkdir -p $@
 
-penguin_test_threshold = 60
+# At last check, the penguin performance test was running on GitHub's runner
+# (Ubuntu 22.04.4 LTS, runner version 2.317.0) in just under 30s. GitHub
+# sometimes changes the runner version, however, thus if we set the threshold
+# too low we might get a failure. The threshold below is about 10s more than the time
+# it takes on my laptop (while plugged).
+penguin_test_threshold = 50
 num_penguin_rows = 100000
 penguin_command_sqlite = ./valve --assume-yes load src/schema/table.tsv --initial-load penguins.db
 penguin_command_pg = ./valve --assume-yes load src/schema/table.tsv $(pg_connect_string)
@@ -200,9 +205,16 @@ $(guess_test_db): valve guess_test_data $(guess_test_dir)/*.tsv | build $(guess_
 	rm -f $@
 	./$< --assume-yes load $(guess_test_dir)/table.tsv $@
 
+# At last check, the performance test was running on GitHub's runner
+# (Ubuntu 22.04.4 LTS, runner version 2.317.0) in just over 20s. GitHub
+# sometimes changes the runner version, however, thus if we set the threshold
+# too low we might get a failure. The threshold below is about 10s more than the time
+# it takes using postgresql on my laptop (while plugged), and about 15s more than it takes
+# using sqlite.
+perf_test_threshold = 45
 perf_test_dir = test/perf_test_data
 perf_test_db = build/valve_perf.db
-num_perf_test_rows = 1000
+num_perf_test_rows = 10000
 perf_test_error_rate = 5
 
 $(perf_test_dir)/ontology:
@@ -212,19 +224,22 @@ $(perf_test_dir)/ontology:
 perf_test_data: test/generate_random_test_data.py valve confirm_overwrite.sh $(perf_test_dir)/*.tsv | $(perf_test_dir)/ontology
 	./confirm_overwrite.sh $(perf_test_dir)/ontology
 	rm -f $(perf_test_dir)/ontology/*.tsv
-	./$< $$(date +"%s") $(num_perf_test_rows) $(perf_test_error_rate) $(perf_test_dir)/table.tsv $|
+	./$< 0 $(num_perf_test_rows) $(perf_test_error_rate) $(perf_test_dir)/table.tsv $|
 
 $(perf_test_db): valve perf_test_data $(perf_test_dir)/*.tsv | build $(perf_test_dir)/ontology
 	rm -f $@
-	time -p ./$< --verbose load $(perf_test_dir)/table.tsv --initial-load $@
+	timeout $(perf_test_threshold) time -p ./$< --assume-yes --verbose load $(perf_test_dir)/table.tsv --initial-load $@ || \
+		(echo "Performance test (SQLite) took longer than $(perf_test_threshold) seconds." && false)
+
 
 .PHONY: sqlite_perf_test
 sqlite_perf_test: $(perf_test_db) | test/output
 	time -p scripts/export_messages.py $< $| $(tables_to_test)
 
 .PHONY: pg_perf_test
 pg_perf_test: valve $(perf_test_dir)/ontology | test/output
-	time -p ./$< --verbose load $(perf_test_dir)/table.tsv $(pg_connect_string)
+	timeout $(perf_test_threshold) time -p ./$< --assume-yes --verbose load $(perf_test_dir)/table.tsv $(pg_connect_string) || \
+        (echo "Performance test (PostgreSQL) took longer than $(perf_test_threshold) seconds." && false)
 	time -p scripts/export_messages.py $(pg_connect_string) $| $(tables_to_test)
 
 .PHONY: perf_test

src/lib.rs

@@ -1,32 +1,8 @@
-//! <!-- Please do not edit README.md directly. To generate a new readme from the crate documentation
-//!      in src/lib.rs, install cargo-readme using `cargo install cargo-readme` and then run:
-//!      `cargo readme > README.md` -->
-//!
-//! # valve.rs
+//! # ontodev/valve.rs
 //! A lightweight validation engine written in rust.
 //!
 //! ## API
 //! See [valve]
-//!
-//! ## Command line usage
-//! Run:
-//! ```
-//! valve --help
-//! ```
-//! to see command line options.
-//!
-//! ## Logging
-//! By default Valve only logs error messages. To also enable warning and information messages,
-//! set the environment variable `RUST_LOG` to the minimum logging level desired for ontodev_valve:
-//! `debug`, `info`, `warn`, or `error`.
-//! For instance:
-//! ```
-//! export RUST_LOG="ontodev_valve=info"
-//! ```
-//! For further information see the [Rust Cookbook](https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html).
-//!
-//! ## Python bindings
-//! See [valve.py](https://github.com/ontodev/valve.py)
 
 #[macro_use]
 extern crate lalrpop_util;
@@ -62,10 +38,7 @@ pub static MOVE_INTERVAL: u32 = 1000;
 pub static PRINTF_RE: &str = r#"^%.*([\w%])$"#;
 
 /// The size of the datatype validation cache.
-static DT_CACHE_SIZE: usize = 10000;
-
-/// The size of the foreign key validation cache.
-static FKEY_CACHE_SIZE: usize = 10000;
+pub static DT_CACHE_SIZE: usize = 10000;
 
 // Note that SQL_PARAM must be a 'word' (from the point of view of regular expressions) since in the
 // local_sql_syntax() function below we are matchng against it using '\b' which represents a word

src/main.rs

@@ -22,7 +22,7 @@ static SAVE_DIR_HELP: &str = "Save tables to DIR instead of to their configured
           long_about = None)]
 struct Cli {
     /// Use this option with caution. When set, Valve will not not ask the user for confirmation
-    /// before executing potentially destructive operations.
+    /// before executing potentially destructive operations on the database and/or table files.
     #[arg(long, action = ArgAction::SetTrue)]
     assume_yes: bool,
 

src/tests.rs

@@ -130,6 +130,25 @@ async fn test_insert_2(valve: &Valve) -> Result<()> {
     Ok(())
 }
 
+async fn test_insert_3(valve: &Valve) -> Result<()> {
+    eprint!("Running test_insert_3() ... ");
+
+    let row = json!({
+        "id": "BFO:0000099",
+        "label": "jafar",
+        "parent": "mar",
+        "source": "COB",
+        "type": "owl:Class",
+    });
+    let (_new_row_num, _new_row) = valve.insert_row("table3", row.as_object().unwrap()).await?;
+
+    // The result of this insertion is that the tree:foreign error message will be resolved
+    // for table3 row 5 column parent: "Value 'jafar' of column parent is not in column label"
+
+    eprintln!("done.");
+    Ok(())
+}
+
 async fn test_dependencies(valve: &Valve) -> Result<()> {
     eprint!("Running test_dependencies() ... ");
 
@@ -648,55 +667,43 @@ async fn test_modes(valve: &Valve) -> Result<()> {
 
     let result = valve.insert_row("readonly1", &readonly_row).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Inserting to table 'readonly1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Inserting to table 'readonly1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
     let result = valve.insert_row("view1", &view_row).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Inserting to table 'view1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Inserting to table 'view1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
     let result = valve.update_row("readonly1", &1, &readonly_row).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Updating table 'readonly1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Updating table 'readonly1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
     let result = valve.update_row("view1", &1, &view_row).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Updating table 'view1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Updating table 'view1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
     let result = valve.delete_row("readonly1", &1).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Deleting from table 'readonly1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Deleting from table 'readonly1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
     let result = valve.delete_row("view1", &1).await;
     match result {
-        Err(e) => assert_eq!(
-            format!("{:?}", e),
-            r#"InputError("Deleting from table 'view1' is not allowed")"#,
-        ),
+        Err(e) => assert!(format!("{:?}", e)
+            .starts_with(r#"InputError("Deleting from table 'view1' is not allowed")"#)),
         _ => assert!(false, "Expected an error result but got an OK result"),
     };
 
@@ -977,6 +984,7 @@ pub async fn run_api_tests(valve: &Valve) -> Result<()> {
     test_insert_1(&valve).await?;
     test_update_2(&valve).await?;
     test_insert_2(&valve).await?;
+    test_insert_3(&valve).await?;
     test_dependencies(&valve).await?;
     test_undo_redo(&valve).await?;
     test_randomized_api_test_with_undo_redo(&valve).await?;