Integrate preprocessor with LSP

calyx-frontend/src/ast.rs

@@ -21,12 +21,15 @@ pub struct NamespaceDef {
 impl NamespaceDef {
     /// Construct a namespace from a file or the input stream.
     /// If no file is provided, the input stream must be a TTY.
-    pub fn construct(file: &Option<PathBuf>) -> CalyxResult<Self> {
+    pub fn construct<F: Fn(&str) -> CalyxResult<String>>(
+        file: &Option<PathBuf>,
+        preprocess: &F,
+    ) -> CalyxResult<Self> {
         match file {
-            Some(file) => parser::CalyxParser::parse_file(file),
+            Some(file) => parser::CalyxParser::parse_file(file, preprocess),
             None => {
                 if atty::isnt(Stream::Stdin) {
-                    parser::CalyxParser::parse(std::io::stdin())
+                    parser::CalyxParser::parse(std::io::stdin(), preprocess)
                 } else {
                     Err(Error::invalid_file(
                         "No file provided and terminal not a TTY".to_string(),
@@ -38,7 +41,10 @@ impl NamespaceDef {
 
     /// Construct a namespace from a definition using a string.
     pub fn construct_from_str(inp: &str) -> CalyxResult<Self> {
-        parser::CalyxParser::parse(inp.as_bytes())
+        parser::CalyxParser::parse(
+            inp.as_bytes(),
+            &parser::default_preprocessor,
+        )
     }
 }
 

calyx-frontend/src/parser.rs

@@ -47,7 +47,10 @@ pub struct CalyxParser;
 
 impl CalyxParser {
     /// Parse a Calyx program into an AST representation.
-    pub fn parse_file(path: &Path) -> CalyxResult<ast::NamespaceDef> {
+    pub fn parse_file<F: Fn(&str) -> CalyxResult<String>>(
+        path: &Path,
+        preprocess: &F,
+    ) -> CalyxResult<ast::NamespaceDef> {
         let time = std::time::Instant::now();
         let content = &fs::read(path).map_err(|err| {
             calyx_utils::Error::invalid_file(format!(
@@ -56,7 +59,7 @@ impl CalyxParser {
             ))
         })?;
         // Add a new file to the position table
-        let string_content = std::str::from_utf8(content)?.to_string();
+        let string_content = preprocess(std::str::from_utf8(content)?)?;
         let file = GlobalPositionTable::as_mut()
             .add_file(path.to_string_lossy().to_string(), string_content);
         let user_data = UserData { file };
@@ -94,13 +97,17 @@ impl CalyxParser {
         Ok(out)
     }
 
-    pub fn parse<R: Read>(mut r: R) -> CalyxResult<ast::NamespaceDef> {
+    pub fn parse<R: Read, F: Fn(&str) -> CalyxResult<String>>(
+        mut r: R,
+        preprocess: &F,
+    ) -> CalyxResult<ast::NamespaceDef> {
         let mut buf = String::new();
         r.read_to_string(&mut buf).map_err(|err| {
             calyx_utils::Error::invalid_file(format!(
                 "Failed to parse buffer: {err}",
             ))
         })?;
+        buf = preprocess(&buf)?;
         // Save the input string to the position table
         let file =
             GlobalPositionTable::as_mut().add_file("<stdin>".to_string(), buf);
@@ -1329,3 +1336,7 @@ impl CalyxParser {
         ))
     }
 }
+
+pub fn default_preprocessor(text: &str) -> CalyxResult<String> {
+    Ok(text.into())
+}

calyx-frontend/src/workspace.rs

@@ -140,9 +140,22 @@ impl Workspace {
         file: &Option<PathBuf>,
         lib_path: &Path,
     ) -> CalyxResult<Self> {
-        Self::construct_with_all_deps::<false>(
+        Self::construct_with_all_deps::<false, _>(
             file.iter().cloned().collect(),
             lib_path,
+            parser::default_preprocessor,
+        )
+    }
+
+    pub fn construct_with_preprocessor<F: Fn(&str) -> CalyxResult<String>>(
+        file: &Option<PathBuf>,
+        lib_path: &Path,
+        preprocess: F,
+    ) -> CalyxResult<Self> {
+        Self::construct_with_all_deps::<false, _>(
+            file.iter().cloned().collect(),
+            lib_path,
+            preprocess,
         )
     }
 
@@ -152,9 +165,10 @@ impl Workspace {
         file: &Option<PathBuf>,
         lib_path: &Path,
     ) -> CalyxResult<Self> {
-        Self::construct_with_all_deps::<true>(
+        Self::construct_with_all_deps::<true, _>(
             file.iter().cloned().collect(),
             lib_path,
+            parser::default_preprocessor,
         )
     }
 
@@ -236,13 +250,17 @@ impl Workspace {
     /// Construct the Workspace using the given files and all their dependencies.
     /// If SHALLOW is true, then parse imported components as declarations and not added to the workspace components.
     /// If in doubt, set SHALLOW to false.
-    pub fn construct_with_all_deps<const SHALLOW: bool>(
+    pub fn construct_with_all_deps<
+        const SHALLOW: bool,
+        F: Fn(&str) -> CalyxResult<String>,
+    >(
         mut files: Vec<PathBuf>,
         lib_path: &Path,
+        preprocess: F,
     ) -> CalyxResult<Self> {
         // Construct initial namespace. If `files` is empty, then we're reading from the standard input.
         let first = files.pop();
-        let ns = NamespaceDef::construct(&first)?;
+        let ns = NamespaceDef::construct(&first, &preprocess)?;
         let parent_path = first
             .as_ref()
             .map(|p| Self::get_parent(p))
@@ -291,7 +309,7 @@ impl Workspace {
             if already_imported.contains(&p) {
                 continue;
             }
-            let ns = parser::CalyxParser::parse_file(&p)?;
+            let ns = parser::CalyxParser::parse_file(&p, &preprocess)?;
             let parent = Self::get_parent(&p);
 
             let mut deps = ws.merge_namespace(

calyx-lsp/Cargo.toml

@@ -37,6 +37,7 @@ tower-lsp = "0.20"
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 tree-sitter = "0.20"
+preprocessor = { path = "../tools/preprocessor" }
 
 [dependencies.tokio]
 version = "1"

calyx-lsp/README.md

@@ -8,3 +8,14 @@ Build the repo with `cargo build`. This uses `build.rs` to build and link the tr
 cd ~/.local/bin
 ln -s $calyx_repo/target/debug/calyx-lsp calyx-lsp
 ```
+
+
+### Note from Ethan:
+
+I made a hacky install script if you want to use it:
+```shell
+yes | ./install.sh
+```
+
+You can, of course, customize what it does by omitting `yes | ` and answering "No" to some of the defaults.
+(Remember to `sudo rm` the original location first.)

calyx-lsp/install.sh

@@ -0,0 +1,49 @@
+INSTALL_LOC="/usr/local/bin"
+
+echo "You are installing the calyx-lsp program"
+echo "========================================"
+
+echo "0. Checks:"
+grep -q 'name = "calyx"' ../Cargo.toml 1>/dev/null 2>/dev/null
+if [ $? -ne 0 ] ; then
+    echo "   => Please run this install script from the calyx repository ❌"
+    exit 1
+else
+    echo "   => You are installing this from the calyx repository ✅"
+fi
+
+if which calyx-lsp >/dev/null; then
+    echo "   => calyx-lsp is already installed under $(which calyx-lsp) ❌"
+    exit 1
+else
+    echo "   => This is a fresh install ✅"
+fi
+
+echo
+echo "1. Configure install location:"
+echo "   => Default: under '$INSTALL_LOC'"
+printf "   => Are you ok with this? (y/n) "
+read answer
+if [ "$answer" != "y" ]; then
+    printf "   => Please enter the new install location: "
+    read INSTALL_LOC
+fi
+echo
+echo "2. Confirm install location:"
+printf "   => Confirm that you want to install under '$INSTALL_LOC' (y/n) "
+read answer
+if [ "$answer" != "y" ]; then
+    echo
+    echo "Exiting..."
+    exit 0
+fi
+echo
+cargo build --manifest-path Cargo.toml || exit 1
+OLD_PWD=`pwd`
+cd ..
+REPO=`pwd`
+cd $INSTALL_LOC
+sudo ln -s "$REPO/target/debug/calyx-lsp" calyx-lsp \
+    && echo "Installation was successful!" \
+    || sudo rm -f calyx-lsp
+cd $OLD_PWD

calyx-lsp/src/diagnostic.rs

@@ -2,6 +2,8 @@ use std::path::Path;
 
 use resolve_path::PathResolveExt;
 
+use crate::utils;
+
 pub struct Diagnostic;
 
 /// A Calyx error message
@@ -17,9 +19,10 @@ pub struct CalyxError {
 impl Diagnostic {
     /// Run the `calyx` compiler on `path` with libraries at `lib_path`
     pub fn did_save(path: &Path, lib_path: &Path) -> Vec<CalyxError> {
-        calyx_frontend::Workspace::construct(
+        calyx_frontend::Workspace::construct_with_preprocessor(
             &Some(path.to_path_buf()),
             lib_path.resolve().as_ref(),
+            utils::apply_preprocessor,
         )
         .and_then(calyx_ir::from_ast::ast_to_ir)
         // TODO: call well-formed pass

calyx-lsp/src/document.rs

@@ -12,6 +12,7 @@ use tree_sitter as ts;
 use crate::convert::{Contains, Point, Range};
 use crate::log;
 use crate::ts_utils::ParentUntil;
+use crate::utils;
 use crate::{tree_sitter_calyx, Config};
 
 pub struct Document {
@@ -96,7 +97,8 @@ impl Document {
 
     /// Update the document with a with entirely new text.
     pub fn parse_whole_text(&mut self, text: &str) {
-        self.text = text.to_string();
+        let text = utils::apply_preprocessor(text).unwrap_or(text.to_string());
+        self.text = text.clone();
         self.tree = self.parser.parse(text, None);
         self.update_component_map();
         log::Debug::update(

calyx-lsp/src/main.rs

@@ -6,6 +6,7 @@ mod goto_definition;
 mod log;
 mod query_result;
 mod ts_utils;
+mod utils;
 
 use std::collections::HashMap;
 use std::fs;

calyx-lsp/src/utils.rs

@@ -0,0 +1,14 @@
+use calyx_utils::{CalyxResult, Error};
+
+pub fn apply_preprocessor(text: &str) -> CalyxResult<String> {
+    let mut ctx = preprocessor::Context::new();
+    let result = text
+        .lines()
+        .map(|line| {
+            ctx.process(line.into())
+                .map_err(|err| Error::misc(err.to_string()))
+        })
+        .collect::<CalyxResult<Vec<_>>>()?;
+
+    Ok(result.join("\n"))
+}

web/rust/src/lib.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::unused_unit)]
 use std::path::PathBuf;
 
-use calyx_frontend as frontend;
+use calyx_frontend::{self as frontend, parser};
 use calyx_ir as ir;
 use calyx_opt::pass_manager::PassManager;
 use calyx_utils::{CalyxResult, Error};
@@ -34,6 +34,7 @@ fn compile(
 
     let ns = frontend::parser::CalyxParser::parse(
         (library.to_string() + "\n" + namespace).as_bytes(),
+        &parser::default_preprocessor,
     )?;
     let ws = ws_from_ns(ns)?;