fix: emit scan errors

Author: psteinroe

Cargo.lock

@@ -15,9 +15,9 @@ version              = "0.0.0"
 regex = "1.9.1"
 
 pg_query.workspace           = true
+pglt_diagnostics.workspace   = true
 pglt_lexer_codegen.workspace = true
 
-cstree              = { version = "0.12.0", features = ["derive"] }
 text-size.workspace = true
 
 [dev-dependencies]

crates/pglt_lexer/Cargo.toml

@@ -107,15 +107,14 @@ fn whitespace_tokens(input: &str) -> VecDeque<Token> {
 /// Turn a string of potentially valid sql code into a list of tokens, including their range in the source text.
 ///
 /// The implementation is primarily using libpg_querys `scan` method, and fills in the gaps with tokens that are not parsed by the library, e.g. whitespace.
-pub fn lex(text: &str) -> Vec<Token> {
+pub fn lex(text: &str) -> pg_query::Result<Vec<Token>> {
     let mut whitespace_tokens = whitespace_tokens(text);
 
     // tokens from pg_query.rs
-    let mut pglt_query_tokens = match pg_query::scan(text) {
-        Ok(scanned) => VecDeque::from(scanned.tokens),
-        // this _should_ never fail
-        _ => panic!("pg_query::scan failed"),
-    };
+    let mut pglt_query_tokens = pg_query::scan(text)?
+        .tokens
+        .into_iter()
+        .collect::<VecDeque<_>>();
 
     // merge the two token lists
     let mut tokens: Vec<Token> = Vec::new();
@@ -173,7 +172,7 @@ pub fn lex(text: &str) -> Vec<Token> {
         );
     }
 
-    tokens
+    Ok(tokens)
 }
 
 #[cfg(test)]
@@ -183,36 +182,36 @@ mod tests {
     #[test]
     fn test_special_chars() {
         let input = "insert into c (name, full_name) values ('Å', 1);";
-        let tokens = lex(input);
+        let tokens = lex(input).unwrap();
         assert!(!tokens.is_empty());
     }
 
     #[test]
     fn test_tab_tokens() {
         let input = "select\t1";
-        let tokens = lex(input);
+        let tokens = lex(input).unwrap();
         assert_eq!(tokens[1].kind, SyntaxKind::Tab);
     }
 
     #[test]
     fn test_newline_tokens() {
         let input = "select\n1";
-        let tokens = lex(input);
+        let tokens = lex(input).unwrap();
         assert_eq!(tokens[1].kind, SyntaxKind::Newline);
     }
 
     #[test]
     fn test_whitespace_tokens() {
         let input = "select 1";
-        let tokens = lex(input);
+        let tokens = lex(input).unwrap();
         assert_eq!(tokens[1].kind, SyntaxKind::Whitespace);
     }
 
     #[test]
     fn test_lexer() {
         let input = "select 1; \n -- some comment \n select 2\t";
 
-        let tokens = lex(input);
+        let tokens = lex(input).unwrap();
         let mut tokens_iter = tokens.iter();
 
         let token = tokens_iter.next().unwrap();

crates/pglt_lexer/src/lib.rs

@@ -13,7 +13,6 @@ pub fn lexer_codegen(_item: proc_macro::TokenStream) -> proc_macro::TokenStream
 
     quote! {
         use pg_query::{protobuf, protobuf::ScanToken, protobuf::Token, NodeEnum, NodeRef};
-        use cstree::Syntax;
 
         #syntax_kind
     }

crates/pglt_lexer_codegen/src/lib.rs

@@ -26,7 +26,7 @@ pub fn syntax_kind_mod(proto_file: &ProtoFile) -> proc_macro2::TokenStream {
         /// An u32 enum of all valid syntax elements (nodes and tokens) of the postgres
         /// sql dialect, and a few custom ones that are not parsed by pg_query.rs, such
         /// as `Whitespace`.
-        #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Syntax)]
+        #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
         #[repr(u32)]
         pub enum SyntaxKind {
             #(#unique_enum_variants),*,

crates/pglt_lexer_codegen/src/syntax_kind.rs

@@ -12,9 +12,11 @@ version              = "0.0.0"
 
 
 [dependencies]
-pglt_lexer.workspace = true
-text-size.workspace  = true
+pglt_diagnostics         = { workspace = true }
+pglt_lexer.workspace     = true
+pglt_query_ext.workspace = true
+regex.workspace          = true
+text-size.workspace      = true
 
 [dev-dependencies]
-ntest              = "0.9.3"
-pg_query.workspace = true
+ntest = "0.9.3"

crates/pglt_statement_splitter/Cargo.toml

@@ -0,0 +1,80 @@
+use pglt_diagnostics::{Diagnostic, MessageAndDescription};
+use text_size::TextRange;
+
+/// A specialized diagnostic for the statement splitter parser.
+///
+/// Parser diagnostics are always **errors**.
+#[derive(Clone, Debug, Diagnostic, PartialEq)]
+#[diagnostic(category = "syntax", severity = Error)]
+pub struct ParseDiagnostic {
+    /// The location where the error is occurred
+    #[location(span)]
+    span: Option<TextRange>,
+    #[message]
+    #[description]
+    pub message: MessageAndDescription,
+    // if true, the error is fatal and the parsing should stop
+    pub is_fatal: bool,
+}
+
+impl ParseDiagnostic {
+    pub fn new(message: impl Into<String>, range: TextRange) -> Self {
+        Self {
+            span: Some(range),
+            message: MessageAndDescription::from(message.into()),
+            is_fatal: false,
+        }
+    }
+
+    pub fn from_pg_query_err(err: pglt_query_ext::Error, input: &str) -> Vec<Self> {
+        let err_msg = err.to_string();
+        let re = regex::Regex::new(r#"at or near "(.*?)""#).unwrap();
+        let mut diagnostics = Vec::new();
+
+        for captures in re.captures_iter(&err_msg) {
+            if let Some(matched) = captures.get(1) {
+                let search_term = matched.as_str();
+                for (idx, _) in input.match_indices(search_term) {
+                    let from = idx;
+                    let to = from + search_term.len();
+                    diagnostics.push(ParseDiagnostic {
+                        span: Some(TextRange::new(
+                            from.try_into().unwrap(),
+                            to.try_into().unwrap(),
+                        )),
+                        message: MessageAndDescription::from(err_msg.clone()),
+                        is_fatal: true,
+                    });
+                }
+            }
+        }
+
+        if diagnostics.is_empty() {
+            diagnostics.push(ParseDiagnostic {
+                span: None,
+                message: MessageAndDescription::from(err_msg),
+                is_fatal: true,
+            });
+        }
+
+        diagnostics
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use pglt_lexer::lex;
+
+    use super::*;
+
+    #[test]
+    fn failing_lexer() {
+        let input =
+            "select 1443ddwwd33djwdkjw13331333333333; select 1443ddwwd33djwdkjw13331333333333;";
+        let err = lex(input).unwrap_err();
+
+        let diagnostics = ParseDiagnostic::from_pg_query_err(err, input);
+        assert_eq!(diagnostics.len(), 2);
+        assert!(diagnostics.iter().all(|d| d.is_fatal));
+    }
+}

crates/pglt_statement_splitter/src/diagnostics.rs

@@ -1,13 +1,24 @@
 //! Postgres Statement Splitter
 //!
 //! This crate provides a function to split a SQL source string into individual statements.
+pub mod diagnostics;
 mod parser;
-mod syntax_error;
 
+use diagnostics::ParseDiagnostic;
 use parser::{source, Parse, Parser};
 
 pub fn split(sql: &str) -> Parse {
-    let mut parser = Parser::new(sql);
+    let tokens = match pglt_lexer::lex(sql) {
+        Ok(tokens) => tokens,
+        Err(e) => {
+            return Parse {
+                ranges: Vec::new(),
+                errors: ParseDiagnostic::from_pg_query_err(e, sql),
+            };
+        }
+    };
+
+    let mut parser = Parser::new(tokens);
 
     source(&mut parser);
 
@@ -16,9 +27,9 @@ pub fn split(sql: &str) -> Parse {
 
 #[cfg(test)]
 mod tests {
+    use diagnostics::ParseDiagnostic;
     use ntest::timeout;
     use pglt_lexer::SyntaxKind;
-    use syntax_error::SyntaxError;
     use text_size::TextRange;
 
     use super::*;
@@ -64,7 +75,7 @@ mod tests {
             self
         }
 
-        fn expect_errors(&self, expected: Vec<SyntaxError>) -> &Self {
+        fn expect_errors(&self, expected: Vec<ParseDiagnostic>) -> &Self {
             assert_eq!(
                 self.parse.errors.len(),
                 expected.len(),
@@ -82,6 +93,13 @@ mod tests {
         }
     }
 
+    #[test]
+    fn failing_lexer() {
+        let input = "select 1443ddwwd33djwdkjw13331333333333";
+        let res = split(input);
+        assert!(res.errors.iter().any(|d| d.is_fatal));
+    }
+
     #[test]
     #[timeout(1000)]
     fn basic() {
@@ -114,7 +132,7 @@ mod tests {
     fn insert_expect_error() {
         Tester::from("\ninsert select 1\n\nselect 3")
             .expect_statements(vec!["insert select 1", "select 3"])
-            .expect_errors(vec![SyntaxError::new(
+            .expect_errors(vec![ParseDiagnostic::new(
                 format!("Expected {:?}", SyntaxKind::Into),
                 TextRange::new(8.into(), 14.into()),
             )]);