Skip to content

Commit

Permalink
remove the distinction between 'SQLite type' and 'PostgreSQL type'
Browse files Browse the repository at this point in the history
  • Loading branch information
@lmcmicu
lmcmicu committed Jan 5, 2024
1 parent 7e6bac7 commit c8df0c9
Show file tree
Hide file tree
Showing 11 changed files with 112 additions and 141 deletions.
12 changes: 6 additions & 6 deletions scripts/guess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,14 +155,14 @@ def get_higher_datatypes(datatype_hierarchies, universals, depth):

def get_sql_type(config, datatype):
"""Given the config map and the name of a datatype, climb the datatype tree (as required),
and return the first 'SQLite type' found."""
and return the first 'SQL type' found."""
if "datatype" not in config:
print("Missing datatypes in config")
sys.exit(1)
if datatype not in config["datatype"]:
return None
if config["datatype"][datatype].get("SQLite type"):
return config["datatype"][datatype]["SQLite type"]
if config["datatype"][datatype].get("SQL type"):
return config["datatype"][datatype]["SQL type"]
return get_sql_type(config, config["datatype"][datatype].get("parent"))


Expand Down Expand Up @@ -260,9 +260,9 @@ def is_match(datatype):
# If the datatype has no associated condition then it matches anything:
if not datatype.get("condition"):
return True
# If the SQLite type is NULL this datatype is ruled out:
sqlite_type = datatype.get("SQLite type")
if sqlite_type and sqlite_type.casefold() == "null":
# If the SQL type is NULL this datatype is ruled out:
sql_type = datatype.get("SQL type")
if sql_type and sql_type.casefold() == "null":
return False

condition = get_compiled_condition(datatype["condition"], config["parser"])
Expand Down
46 changes: 11 additions & 35 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ static MULTI_THREADED: bool = true;
static SQL_PARAM: &str = "VALVEPARAM";

lazy_static! {
static ref PG_SQL_TYPES: Vec<&'static str> =
vec!["text", "varchar", "numeric", "integer", "real"];
static ref SL_SQL_TYPES: Vec<&'static str> = vec!["text", "numeric", "integer", "real"];
static ref SQL_TYPES: Vec<&'static str> = vec!["text", "varchar", "numeric", "integer", "real"];
}

/// Aliases for [serde_json::Map](..//serde_json/struct.Map.html)<String, [serde_json::Value](../serde_json/enum.Value.html)>.
Expand Down Expand Up @@ -2098,8 +2096,7 @@ fn read_config_files(
"datatype",
"parent",
"condition",
"SQLite type",
"PostgreSQL type",
"SQL type",
] {
if !row.contains_key(column) || row.get(column) == None {
panic!("Missing required column '{}' reading '{}'", column, path);
Expand All @@ -2112,7 +2109,7 @@ fn read_config_files(
}
}

for column in vec!["parent", "condition", "SQLite type", "PostgreSQL type"] {
for column in vec!["parent", "condition", "SQL type"] {
if row.get(column).and_then(|c| c.as_str()).unwrap() == "" {
row.remove(&column.to_string());
}
Expand Down Expand Up @@ -4354,15 +4351,7 @@ fn get_sql_type(dt_config: &SerdeMap, datatype: &String, pool: &AnyPool) -> Opti
return None;
}

let sql_type_column = {
if pool.any_kind() == AnyKind::Sqlite {
"SQLite type"
} else {
"PostgreSQL type"
}
};

if let Some(sql_type) = dt_config.get(datatype).and_then(|d| d.get(sql_type_column)) {
if let Some(sql_type) = dt_config.get(datatype).and_then(|d| d.get("SQL type")) {
return Some(sql_type.as_str().and_then(|s| Some(s.to_string())).unwrap());
}

Expand Down Expand Up @@ -4956,26 +4945,13 @@ fn get_table_ddl(
}
};

if pool.any_kind() == AnyKind::Postgres {
if !PG_SQL_TYPES.contains(&short_sql_type.to_lowercase().as_str()) {
panic!(
"Unrecognized PostgreSQL SQL type '{}' for datatype: '{}'. \
Accepted SQL types for PostgreSQL are: {}",
sql_type,
row.get("datatype").and_then(|d| d.as_str()).unwrap(),
PG_SQL_TYPES.join(", ")
);
}
} else {
if !SL_SQL_TYPES.contains(&short_sql_type.to_lowercase().as_str()) {
panic!(
"Unrecognized SQLite SQL type '{}' for datatype '{}'. \
Accepted SQL datatypes for SQLite are: {}",
sql_type,
row.get("datatype").and_then(|d| d.as_str()).unwrap(),
SL_SQL_TYPES.join(", ")
);
}
if !SQL_TYPES.contains(&short_sql_type.to_lowercase().as_str()) {
panic!(
"Unrecognized SQL type '{}' for datatype: '{}'. Accepted SQL types are: {}",
sql_type,
row.get("datatype").and_then(|d| d.as_str()).unwrap(),
SQL_TYPES.join(", ")
);
}

let column_name = row.get("column").and_then(|s| s.as_str()).unwrap();
Expand Down
3 changes: 1 addition & 2 deletions test/guess_test_data/column.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ datatype transform empty word
datatype condition empty datatype_condition
datatype structure empty trimmed_line
datatype description empty trimmed_text
datatype SQLite type empty datatype_name
datatype PostgreSQL type empty datatype_name
datatype SQL type empty datatype_name
datatype RDF type empty datatype_name
datatype HTML type empty datatype_name
rule table table_name
Expand Down
3 changes: 1 addition & 2 deletions test/guess_test_data/column_expected.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ datatype transform empty word
datatype condition empty datatype_condition
datatype structure empty trimmed_line
datatype description empty trimmed_text
datatype SQLite type empty datatype_name
datatype PostgreSQL type empty datatype_name
datatype SQL type empty datatype_name
datatype RDF type empty datatype_name
datatype HTML type empty datatype_name
rule table table_name
Expand Down
44 changes: 22 additions & 22 deletions test/guess_test_data/datatype.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
datatype parent transform condition structure description SQLite type PostgreSQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
datatype parent transform condition structure description SQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
3 changes: 1 addition & 2 deletions test/perf_test_data/column.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ datatype transform empty word
datatype condition empty datatype_condition
datatype structure empty trimmed_line
datatype description empty trimmed_text
datatype SQLite type empty datatype_name
datatype PostgreSQL type empty datatype_name
datatype SQL type empty datatype_name
datatype RDF type empty datatype_name
datatype HTML type empty datatype_name
rule table table_name
Expand Down
44 changes: 22 additions & 22 deletions test/perf_test_data/datatype.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
datatype parent transform condition structure description SQLite type PostgreSQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
datatype parent transform condition structure description SQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
3 changes: 1 addition & 2 deletions test/random_test_data/column.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ datatype transform empty word
datatype condition empty datatype_condition
datatype structure empty trimmed_line
datatype description empty trimmed_text
datatype SQLite type empty datatype_name
datatype PostgreSQL type empty datatype_name
datatype SQL type empty datatype_name
datatype RDF type empty datatype_name
datatype HTML type empty datatype_name
rule table table_name
Expand Down
44 changes: 22 additions & 22 deletions test/random_test_data/datatype.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
datatype parent transform condition structure description SQLite type PostgreSQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
datatype parent transform condition structure description SQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
3 changes: 1 addition & 2 deletions test/src/column.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ datatype transform empty word
datatype condition empty datatype_condition
datatype structure empty trimmed_line
datatype description empty trimmed_text
datatype SQLite type empty trimmed_line
datatype PostgreSQL type empty trimmed_line
datatype SQL type empty trimmed_line
datatype RDF type empty trimmed_line
datatype HTML type empty datatype_name
rule table table_name
Expand Down
48 changes: 24 additions & 24 deletions test/src/datatype.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
datatype parent transform condition structure description SQLite type PostgreSQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
numeric nonspace match(/-?\d+(\.\d+)?/) a positive or negative number NUMERIC NUMERIC
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
real nonspace match(/-?\d+(\.\d+)?/) a positive or negative real number REAL REAL
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore
datatype parent transform condition structure description SQL type RDF type HTML type
CURIE nonspace match(/\S+:\S+/) concat(prefix, ":", suffix) a Compact URI CURIE
IRI nonspace exclude(/\s/) an Internationalized Resource Identifier IRI
column_name trimmed_line match(/\S([^\n]*\S)*/) a column name
datatype_condition line exclude(/\n/) a datatype condition specification
datatype_name word exclude(/\W/) a datatype name
description trimmed_text match(/\S(.*\S)*/) a brief description
empty text equals('') the empty string NULL null
integer nonspace match(/-?\d+/) a positive or negative integer INTEGER
label trimmed_line match(/\S([^\n]*\S)*/)
line text exclude(/\n/) a line of text input
natural_number integer match(/\d+/) a natural number, including zero INTEGER
nonspace trimmed_line exclude(/\s/) text without whitespace
numeric nonspace match(/-?\d+(\.\d+)?/) a positive or negative number NUMERIC
path line exclude(/\n/) a path to a file
prefix word exclude(/\W/) a prefix for a CURIE
real nonspace match(/-?\d+(\.\d+)?/) a positive or negative real number REAL
suffix word exclude(/\W/) a suffix for a CURIE
table_name word exclude(/\W/) a table name
table_type word lowercase in('table', 'column', 'datatype') a table type
text any text TEXT xsd:string textarea
trimmed_line line match(/\S([^\n]*\S)*/) a line of text that does not begin or end with whitespace
trimmed_text text exclude(/^\s+|\s+$/) text that does not begin or end with whitespace
word nonspace exclude(/\W/) a single word: letters, numbers, underscore

0 comments on commit c8df0c9

Please sign in to comment.