feat: saving and loading docs with holes

data/json_lang.ron

@@ -3,6 +3,11 @@
 LanguageSpec(
     name: "json",
     file_extensions: [".json"],
+    hole_syntax: Some(HoleSyntax(
+        invalid: "SYNLESS_HOLE_6CB3433C86C14E599F9F12637A47F6DA",
+        valid: "\"SYNLESS_HOLE_6CB3433C86C14E599F9F12637A47F6DA\"",
+        text: "SYNLESS_HOLE_6CB3433C86C14E599F9F12637A47F6DA",
+    )),
     grammar: GrammarSpec(
         constructs: [
             ConstructSpec(

src/engine/engine.rs

@@ -5,7 +5,7 @@ use super::doc::Doc;
 use super::doc_set::{DocDisplayLabel, DocName, DocSet};
 use super::Settings;
 use crate::language::{Language, LanguageSpec, NotationSetSpec, Storage};
-use crate::parsing::{Parse, ParseError};
+use crate::parsing::{self, Parse, ParseError};
 use crate::pretty_doc::DocRef;
 use crate::style::Base16Color;
 use crate::tree::{Mode, Node};
@@ -249,7 +249,23 @@ impl Engine {
             .parsers
             .get_mut(language_name)
             .ok_or_else(|| error!(Language, "No parser for language {}", language_name))?;
+        let hole_syntax = self
+            .storage
+            .language(language_name)?
+            .hole_syntax(&self.storage)
+            .ok_or_else(|| {
+                error!(
+                    Language,
+                    "No hole syntax for language {}, but it's required for loading from source",
+                    language_name
+                )
+            })?
+            .to_owned();
+
+        let source = &parsing::preprocess(source, &hole_syntax.invalid, &hole_syntax.valid);
         let root_node = parser.parse(&mut self.storage, &doc_name.to_string(), source)?;
+        parsing::postprocess(&mut self.storage, root_node, &hole_syntax.text);
+
         let doc = Doc::new(&self.storage, root_node).bug_msg("Invalid root");
         if !self.doc_set.add_doc(doc_name.clone(), doc) {
             return Err(DocError::DocAlreadyOpen(doc_name).into());

src/language/compiled.rs

@@ -1,5 +1,5 @@
 use super::specs::{
-    AritySpec, ConstructSpec, GrammarSpec, LanguageSpec, NotationSetSpec, SortSpec,
+    AritySpec, ConstructSpec, GrammarSpec, HoleSyntax, LanguageSpec, NotationSetSpec, SortSpec,
 };
 use crate::language::LanguageError;
 use crate::style::{StyleLabel, ValidNotation};
@@ -58,6 +58,9 @@ pub struct LanguageCompiled {
     pub display_notation: NotationSetId,
     /// Load files with these extensions using this language. Must include the `.`.
     pub file_extensions: Vec<String>,
+    pub hole_syntax: Option<HoleSyntax>,
+    pub hole_source_notation: Option<ValidNotation>,
+    pub hole_display_notation: ValidNotation,
 }
 
 #[derive(Debug)]
@@ -92,30 +95,42 @@ pub fn compile_language(language_spec: LanguageSpec) -> Result<LanguageCompiled,
         None
     };
 
+    let (hole_source_notation, hole_display_notation) = {
+        use ppp::notation_constructors::{lit, style};
+
+        let display_notation = style(StyleLabel::Hole, lit(HOLE_LITERAL)).validate().bug();
+        let source_notation = language_spec
+            .hole_syntax
+            .as_ref()
+            .map(|hole_syntax| {
+                style(StyleLabel::Hole, lit(&hole_syntax.invalid))
+                    .validate()
+                    .map_err(|err| {
+                        LanguageError::InvalidHoleNotation(language_spec.name.clone(), err)
+                    })
+            })
+            .transpose()?;
+
+        (source_notation, display_notation)
+    };
+
     Ok(LanguageCompiled {
         name: language_spec.name,
         grammar,
         notation_sets,
         source_notation,
         display_notation,
         file_extensions: language_spec.file_extensions,
+        hole_syntax: language_spec.hole_syntax,
+        hole_source_notation,
+        hole_display_notation,
     })
 }
 
-fn inject_notation_set_builtins(notation_set_spec: &mut NotationSetSpec) {
-    use ppp::notation_constructors::{lit, style};
-    let hole_notation = style(StyleLabel::Hole, lit(HOLE_LITERAL));
-    notation_set_spec
-        .notations
-        .push((HOLE_NAME.to_owned(), hole_notation));
-}
-
 pub(super) fn compile_notation_set(
-    mut notation_set: NotationSetSpec,
+    notation_set: NotationSetSpec,
     grammar: &GrammarCompiled,
 ) -> Result<NotationSetCompiled, LanguageError> {
-    inject_notation_set_builtins(&mut notation_set);
-
     // Put notations in a HashMap, checking for duplicate entries.
     let mut notations_map = HashMap::new();
     for (construct_name, notation) in notation_set.notations {
@@ -144,7 +159,8 @@ pub(super) fn compile_notation_set(
                 )
             })?;
             notations.push(valid_notation);
-        } else {
+        } else if construct.name != HOLE_NAME {
+            // Every construct except for $hole must have a notation.
             return Err(LanguageError::MissingNotation(
                 notation_set.name,
                 construct.name.clone(),

src/language/interface.rs

@@ -4,7 +4,7 @@ use super::compiled::{
 };
 use super::specs::NotationSetSpec;
 use super::storage::Storage;
-use super::LanguageError;
+use super::{HoleSyntax, LanguageError};
 use crate::style::ValidNotation;
 use crate::util::bug;
 
@@ -198,6 +198,18 @@ impl Language {
         Ok(())
     }
 
+    pub fn hole_syntax(self, s: &Storage) -> Option<&HoleSyntax> {
+        s.languages[self.language].hole_syntax.as_ref()
+    }
+
+    pub fn hole_display_notation(self, s: &Storage) -> &ValidNotation {
+        &s.languages[self.language].hole_display_notation
+    }
+
+    pub fn hole_source_notation(self, s: &Storage) -> Option<&ValidNotation> {
+        s.languages[self.language].hole_source_notation.as_ref()
+    }
+
     fn notation_id(self, s: &Storage, notation_set_name: &str) -> Result<usize, LanguageError> {
         if let Some(id) = s.languages[self.language]
             .notation_sets
@@ -301,11 +313,19 @@ impl Construct {
     }
 
     pub fn display_notation(self, s: &Storage) -> &ValidNotation {
-        self.language().display_notation(s).notation(s, self)
+        if self.is_hole(s) {
+            self.language().hole_display_notation(s)
+        } else {
+            self.language().display_notation(s).notation(s, self)
+        }
     }
 
     pub fn source_notation(self, s: &Storage) -> Option<&ValidNotation> {
-        Some(self.language().source_notation(s)?.notation(s, self))
+        if self.is_hole(s) {
+            self.language().hole_source_notation(s)
+        } else {
+            Some(self.language().source_notation(s)?.notation(s, self))
+        }
     }
 
     pub fn is_comment_or_ws(self, s: &Storage) -> bool {

src/language/mod.rs

@@ -8,7 +8,9 @@ use partial_pretty_printer as ppp;
 use std::fmt;
 
 pub use interface::{Arity, Construct, Language};
-pub use specs::{AritySpec, ConstructSpec, GrammarSpec, LanguageSpec, NotationSetSpec, SortSpec};
+pub use specs::{
+    AritySpec, ConstructSpec, GrammarSpec, HoleSyntax, LanguageSpec, NotationSetSpec, SortSpec,
+};
 pub use storage::Storage;
 
 #[derive(thiserror::Error, fmt::Debug)]
@@ -45,6 +47,8 @@ pub enum LanguageError {
     DuplicateNotation(String, String),
     #[error("Invalid notation for construct '{1}' in notation set '{0}':\n{2}")]
     InvalidNotation(String, String, ppp::NotationError),
+    #[error("Invalid notation for holes in language '{0}':\n{1}")]
+    InvalidHoleNotation(String, ppp::NotationError),
 
     // Languages
     #[error("Duplicate name '{0}' used for two languages")]

src/language/specs.rs

@@ -65,4 +65,20 @@ pub struct LanguageSpec {
     pub default_source_notation: Option<String>,
     /// Load files with these extensions using this language. Must include the `.`.
     pub file_extensions: Vec<String>,
+    pub hole_syntax: Option<HoleSyntax>,
+}
+
+/// The syntax to use when saving and loading holes.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct HoleSyntax {
+    /// What to save each hole as in a source file. It should ideally cause a syntax error when
+    /// parsed with the language's standard parser.
+    pub invalid: String,
+    /// Something syntactically valid to convert holes into, before Synless parses it with a
+    /// standard parser for the language.
+    pub valid: String,
+    /// After a `HoleSyntax.valid` is parsed into a texty node, this is the contents of that node.
+    /// It will then be replaced by a hole, completing the cycle.
+    pub text: String,
 }

src/parsing/mod.rs

@@ -2,7 +2,7 @@ mod json_parser;
 
 use crate::language::Storage;
 use crate::tree::Node;
-use crate::util::{error, SynlessError};
+use crate::util::{bug, error, SynlessError};
 use partial_pretty_printer as ppp;
 use std::fmt;
 use std::path::Path;
@@ -20,6 +20,26 @@ pub trait Parse: fmt::Debug {
     ) -> Result<Node, SynlessError>;
 }
 
+/// Convert holes in `source` from `invalid_hole_syntax` to `valid_hole_syntax`, so that they can
+/// be parsed with a standard parser for the language.
+pub fn preprocess(source: &str, invalid_hole_syntax: &str, valid_hole_syntax: &str) -> String {
+    source.replace(invalid_hole_syntax, valid_hole_syntax)
+}
+
+/// Replace every texty node within `root` that contains `hole_text` with a hole.
+pub fn postprocess(s: &mut Storage, root: Node, hole_text: &str) {
+    root.walk_tree(s, |s: &mut Storage, node: Node| {
+        if let Some(text) = node.text(s) {
+            if text.as_str() == hole_text {
+                let hole = Node::new_hole(s, node.language(s));
+                if !node.swap(s, hole) {
+                    bug!("Failed to replace node with hole in parser postprocess()")
+                }
+            }
+        }
+    });
+}
+
 #[derive(Debug)]
 pub struct ParseError {
     pub pos: Option<ppp::Pos>,

src/tree/node.rs

@@ -194,6 +194,10 @@ impl Node {
         s.forest().data(self.0).text.is_some()
     }
 
+    pub fn is_hole(self, s: &Storage) -> bool {
+        s.forest().data(self.0).construct.is_hole(s)
+    }
+
     pub fn can_have_children(self, s: &Storage) -> bool {
         match self.arity(s) {
             Arity::Texty => false,
@@ -499,6 +503,30 @@ impl Node {
         Node(s.node_forest.forest.deep_copy(self.0, &mut clone_data))
     }
 
+    /***********
+     * Walking *
+     ***********/
+
+    /// Invoke `callback` on every descendant of this node, in an unspecified order.
+    pub fn walk_tree(self, s: &mut Storage, mut callback: impl FnMut(&mut Storage, Node)) {
+        // Remaining nodes to walk are `n.first_child()` and `n.next_sibling()` for every `n` in
+        // `stack`.
+        let mut stack = Vec::new();
+        if let Some(node) = self.first_child(s) {
+            stack.push(node);
+        }
+        callback(s, self);
+        while let Some(node) = stack.pop() {
+            if let Some(n) = node.first_child(s) {
+                stack.push(n);
+            }
+            if let Some(n) = node.next_sibling(s) {
+                stack.push(n);
+            }
+            callback(s, node);
+        }
+    }
+
     /*************
      * Debugging *
      *************/

tests/tests.rs

@@ -54,6 +54,7 @@ fn urllang() -> LanguageSpec {
         },
         default_display_notation: "Testlang_notation".to_owned(),
         default_source_notation: None,
+        hole_syntax: None,
         notations: vec![NotationSetSpec {
             name: "Testlang_notation".to_owned(),
             notations: vec![