From 31ad19e9a244a37976d632fffafff0e14a8cc37c Mon Sep 17 00:00:00 2001 From: Sebastian Martinez Date: Tue, 17 Dec 2024 22:42:57 +0100 Subject: [PATCH] Fix syntax highlighting for languages with multiple injections --- Cargo.lock | 22 ++ crates/radicle-types/Cargo.toml | 8 +- crates/radicle-types/src/syntax.rs | 286 +++++++++++++++--------- crates/radicle-types/src/traits/repo.rs | 2 +- public/syntax.css | 11 +- 5 files changed, 214 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 611e528..25e8fd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4134,9 +4134,11 @@ dependencies = [ "tree-sitter-highlight", "tree-sitter-html", "tree-sitter-javascript", + "tree-sitter-jsdoc", "tree-sitter-json", "tree-sitter-md", "tree-sitter-python", + "tree-sitter-regex", "tree-sitter-ruby", "tree-sitter-rust", "tree-sitter-svelte-ng", @@ -6160,6 +6162,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-jsdoc" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3862dfcb1038fc5e7812d7df14190afdeb7e1415288fd5f51f58395f8cb0faf" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-json" version = "0.24.8" @@ -6196,6 +6208,16 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "tree-sitter-regex" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712656f8c262a5a4b7d6026e6246950787d178d613864952554e1516a33ab0c1" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "tree-sitter-ruby" version = "0.23.1" diff --git a/crates/radicle-types/Cargo.toml b/crates/radicle-types/Cargo.toml index b5223df..e9d525c 100644 --- a/crates/radicle-types/Cargo.toml +++ b/crates/radicle-types/Cargo.toml @@ -7,14 +7,13 @@ edition = "2021" anyhow = { version = "1.0.90" } axum = { version = "0.7.5", default-features = false, features = ["json"] } base64 = { version = "0.22.1" } +localtime = { version = "1.3.1" } radicle = { version = "0.14.0", features = ["test"] } radicle-surf = { version = "0.22.1", features = ["serde"] } serde = { version = "1.0.210", features = ["derive"] } serde_json = { version = "1.0.132" } -thiserror = { version = "1.0.65" } -ts-rs = { version = "10.0.0", features = ["serde-json-impl", "no-serde-warnings", "format"] } -localtime = { version = "1.3.1" } tempfile = { version = "3.14.0" } +thiserror = { version = "1.0.65" } tree-sitter-bash = { version = "0.23.3" } tree-sitter-c = { version = "0.23.2" } tree-sitter-css = { version = "0.23.1" } @@ -22,11 +21,14 @@ tree-sitter-go = { version = "0.23.4" } tree-sitter-highlight = { version = "0.24.4" } tree-sitter-html = { version = "0.23.2" } tree-sitter-javascript = { version = "0.23.1" } +tree-sitter-jsdoc = { version = "0.23.2" } tree-sitter-json = { version = "0.24.8" } tree-sitter-md = { version = "0.3.2" } tree-sitter-python = { version = "0.23.4" } +tree-sitter-regex = { version = "0.24.3" } tree-sitter-ruby = { version = "0.23.1" } tree-sitter-rust = { version = "0.23.2" } tree-sitter-svelte-ng = { version = "1.0.2" } tree-sitter-toml-ng = { version = "0.7.0" } tree-sitter-typescript = { version = "0.23.2" } +ts-rs = { version = "10.0.0", features = ["serde-json-impl", "no-serde-warnings", "format"] } diff --git a/crates/radicle-types/src/syntax.rs b/crates/radicle-types/src/syntax.rs index 3024728..782465d 100644 --- a/crates/radicle-types/src/syntax.rs +++ b/crates/radicle-types/src/syntax.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; @@ -14,20 +13,37 @@ use crate as types; /// Highlight groups enabled. const HIGHLIGHTS: &[&str] = &[ "attribute", + "comment", + "comment.documentation", "constant", "constant.builtin", - "comment", "constructor", "declare", + "embedded", + "escape", "export", - "function.builtin", + "float.literal", "function", + "function.builtin", + "function.macro", + "function.method", "identifier", + "indent.and", + "indent.begin", + "indent.branch", + "indent.end", "integer_literal", - "float.literal", "keyword", + "keyword.coroutine", + "keyword.debug", + "keyword.exception", + "keyword.repeat", + "local.definition", + "local.reference", + "local.scope", "label", "module", + "none", "number", "operator", "property", @@ -36,19 +52,22 @@ const HIGHLIGHTS: &[&str] = &[ "punctuation.delimiter", "punctuation.special", "shorthand_property_identifier", + "statement", "string", "string.special", - "statement", "tag", + "tag.delimiter", + "tag.error", + "text", + "text.literal", + "text.title", "type", "type.builtin", + "type.qualifier", "type_annotation", "variable", "variable.builtin", "variable.parameter", - "text", - "text.literal", - "text.title", ]; /// A structure encapsulating an item and styling. @@ -169,6 +188,7 @@ impl Builder { for event in highlights { match event? { ts::HighlightEvent::Source { start, end } => { + println!("highlightEvent::source {start} {end}"); for (i, byte) in code.iter().enumerate().skip(start).take(end - start) { if *byte == b'\n' { self.advance(); @@ -188,6 +208,7 @@ impl Builder { } ts::HighlightEvent::HighlightStart(h) => { let name = HIGHLIGHTS[h.0]; + println!("highlightEvent::HighlightStart {name}"); self.advance(); self.styles.push(name.to_string()); @@ -215,56 +236,102 @@ impl Builder { } /// Syntax highlighter based on `tree-sitter`. -#[derive(Default)] pub struct Highlighter { - configs: HashMap<&'static str, ts::HighlightConfiguration>, + configs: std::collections::HashMap, +} + +impl Default for Highlighter { + fn default() -> Self { + Self::new() + } } impl Highlighter { + pub fn new() -> Self { + let configs: std::collections::HashMap = [ + ("rust", Self::config("rust")), + ("json", Self::config("json")), + ("jsdoc", Self::config("jsdoc")), + ("typescript", Self::config("typescript")), + ("javascript", Self::config("javascript")), + ("markdown", Self::config("markdown")), + ("css", Self::config("css")), + ("go", Self::config("go")), + ("regex", Self::config("regex")), + ("shell", Self::config("shell")), + ("c", Self::config("c")), + ("python", Self::config("python")), + ("svelte", Self::config("svelte")), + ("ruby", Self::config("ruby")), + ("tsx", Self::config("tsx")), + ("html", Self::config("html")), + ("toml", Self::config("toml")), + ] + .into_iter() + .filter_map(|(lang, cfg)| cfg.map(|c| (lang.to_string(), c))) + .collect(); + + Highlighter { configs } + } + /// Highlight a source code file. pub fn highlight(&mut self, path: &Path, code: &[u8]) -> Result, ts::Error> { let mut highlighter = ts::Highlighter::new(); - let Some(config) = self.detect(path, code) else { + // Check for a language if none found return plain lines. + let Some(language) = Self::detect(path, code) else { let Ok(code) = std::str::from_utf8(code) else { return Err(ts::Error::Unknown); }; + println!("Not able to detect language?"); return Ok(code.lines().map(Line::new).collect()); }; + + // Check if there is a configuration if none found return plain lines. + let Some(config) = &mut Self::config(&language) else { + let Ok(code) = std::str::from_utf8(code) else { + return Err(ts::Error::Unknown); + }; + println!("Not found a configuration?"); + return Ok(code.lines().map(Line::new).collect()); + }; + config.configure(HIGHLIGHTS); - let highlights = highlighter.highlight(config, code, None, |_| { - // Language injection callback. - None + let highlights = highlighter.highlight(config, code, None, |language| { + let l: &'static str = std::boxed::Box::leak(language.to_string().into_boxed_str()); + + self.configs.get(l) })?; Builder::default().run(highlights, code) } /// Detect language. - fn detect(&mut self, path: &Path, _code: &[u8]) -> Option<&mut ts::HighlightConfiguration> { + fn detect(path: &Path, _code: &[u8]) -> Option { match path.extension().and_then(|e| e.to_str()) { - Some("rs") => self.config("rust"), - Some("svelte") => self.config("svelte"), - Some("ts" | "js") => self.config("typescript"), - Some("json") => self.config("json"), - Some("sh" | "bash") => self.config("shell"), - Some("md" | "markdown") => self.config("markdown"), - Some("go") => self.config("go"), - Some("c") => self.config("c"), - Some("py") => self.config("python"), - Some("rb") => self.config("ruby"), - Some("tsx") => self.config("tsx"), - Some("html") | Some("htm") | Some("xml") => self.config("html"), - Some("css") => self.config("css"), - Some("toml") => self.config("toml"), + Some("rs") => Some(String::from("rust")), + Some("svelte") => Some(String::from("svelte")), + Some("ts" | "js") => Some(String::from("typescript")), + Some("json") => Some(String::from("json")), + Some("regex") => Some(String::from("regex")), + Some("sh" | "bash") => Some(String::from("shell")), + Some("md" | "markdown") => Some(String::from("markdown")), + Some("go") => Some(String::from("go")), + Some("c") => Some(String::from("c")), + Some("py") => Some(String::from("python")), + Some("rb") => Some(String::from("ruby")), + Some("tsx") => Some(String::from("tsx")), + Some("html") | Some("htm") | Some("xml") => Some(String::from("html")), + Some("css") => Some(String::from("css")), + Some("toml") => Some(String::from("toml")), _ => None, } } /// Get a language configuration. - fn config(&mut self, language: &'static str) -> Option<&mut ts::HighlightConfiguration> { + fn config(language: &str) -> Option { match language { - "rust" => Some(self.configs.entry(language).or_insert_with(|| { + "rust" => Some( ts::HighlightConfiguration::new( tree_sitter_rust::LANGUAGE.into(), language, @@ -272,9 +339,9 @@ impl Highlighter { tree_sitter_rust::INJECTIONS_QUERY, "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "json" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "json" => Some( ts::HighlightConfiguration::new( tree_sitter_json::LANGUAGE.into(), language, @@ -282,27 +349,29 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "typescript" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "javascript" => Some( ts::HighlightConfiguration::new( - tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + tree_sitter_javascript::LANGUAGE.into(), language, - &format!( - "{}\n{}", - tree_sitter_javascript::HIGHLIGHT_QUERY, - tree_sitter_typescript::HIGHLIGHTS_QUERY - ), + tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_javascript::INJECTIONS_QUERY, - &format!( - "{}\n{}", - tree_sitter_javascript::LOCALS_QUERY, - tree_sitter_typescript::LOCALS_QUERY - ), + tree_sitter_javascript::LOCALS_QUERY, + ) + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "typescript" => Some( + ts::HighlightConfiguration::new( + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + language, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + "", + tree_sitter_typescript::LOCALS_QUERY, ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "markdown" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "markdown" => Some( ts::HighlightConfiguration::new( tree_sitter_md::LANGUAGE.into(), language, @@ -310,9 +379,9 @@ impl Highlighter { tree_sitter_md::INJECTION_QUERY_BLOCK, "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "css" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "css" => Some( ts::HighlightConfiguration::new( tree_sitter_css::LANGUAGE.into(), language, @@ -320,9 +389,9 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "go" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "go" => Some( ts::HighlightConfiguration::new( tree_sitter_go::LANGUAGE.into(), language, @@ -330,9 +399,9 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "shell" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "shell" => Some( ts::HighlightConfiguration::new( tree_sitter_bash::LANGUAGE.into(), language, @@ -340,9 +409,9 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "c" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "c" => Some( ts::HighlightConfiguration::new( tree_sitter_c::LANGUAGE.into(), language, @@ -350,9 +419,9 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "python" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "python" => Some( ts::HighlightConfiguration::new( tree_sitter_python::LANGUAGE.into(), language, @@ -360,32 +429,29 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "svelte" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "regex" => Some( + ts::HighlightConfiguration::new( + tree_sitter_regex::LANGUAGE.into(), + language, + tree_sitter_regex::HIGHLIGHTS_QUERY, + "", + "", + ) + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "svelte" => Some( ts::HighlightConfiguration::new( tree_sitter_svelte_ng::LANGUAGE.into(), language, - &format!( - "{}\n{}\n{}", - tree_sitter_javascript::HIGHLIGHT_QUERY, - tree_sitter_typescript::HIGHLIGHTS_QUERY, - tree_sitter_svelte_ng::HIGHLIGHTS_QUERY - ), - &format!( - "{}\n{}", - tree_sitter_javascript::INJECTIONS_QUERY, - tree_sitter_svelte_ng::INJECTIONS_QUERY, - ), - &format!( - "{}\n{}", - tree_sitter_typescript::LOCALS_QUERY, - tree_sitter_svelte_ng::LOCALS_QUERY, - ), + tree_sitter_svelte_ng::HIGHLIGHTS_QUERY, + tree_sitter_svelte_ng::INJECTIONS_QUERY, + tree_sitter_svelte_ng::LOCALS_QUERY, ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "ruby" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "ruby" => Some( ts::HighlightConfiguration::new( tree_sitter_ruby::LANGUAGE.into(), language, @@ -393,27 +459,29 @@ impl Highlighter { "", tree_sitter_ruby::LOCALS_QUERY, ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "tsx" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "jsdoc" => Some( + ts::HighlightConfiguration::new( + tree_sitter_jsdoc::LANGUAGE.into(), + language, + tree_sitter_jsdoc::HIGHLIGHTS_QUERY, + "", + "", + ) + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "tsx" => Some( ts::HighlightConfiguration::new( tree_sitter_typescript::LANGUAGE_TSX.into(), language, - &format!( - "{}\n{}", - tree_sitter_javascript::HIGHLIGHT_QUERY, - tree_sitter_typescript::HIGHLIGHTS_QUERY - ), + tree_sitter_typescript::HIGHLIGHTS_QUERY, tree_sitter_javascript::INJECTIONS_QUERY, - &format!( - "{}\n{}", - tree_sitter_javascript::LOCALS_QUERY, - tree_sitter_typescript::LOCALS_QUERY - ), + tree_sitter_typescript::LOCALS_QUERY, ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "html" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "html" => Some( ts::HighlightConfiguration::new( tree_sitter_html::LANGUAGE.into(), language, @@ -421,9 +489,9 @@ impl Highlighter { tree_sitter_html::INJECTIONS_QUERY, "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), - "toml" => Some(self.configs.entry(language).or_insert_with(|| { + .expect("Highlighter::config: highlight configuration must be valid"), + ), + "toml" => Some( ts::HighlightConfiguration::new( tree_sitter_toml_ng::LANGUAGE.into(), language, @@ -431,8 +499,8 @@ impl Highlighter { "", "", ) - .expect("Highlighter::config: highlight configuration must be valid") - })), + .expect("Highlighter::config: highlight configuration must be valid"), + ), _ => None, } } diff --git a/crates/radicle-types/src/traits/repo.rs b/crates/radicle-types/src/traits/repo.rs index 32d6230..b62d033 100644 --- a/crates/radicle-types/src/traits/repo.rs +++ b/crates/radicle-types/src/traits/repo.rs @@ -154,7 +154,7 @@ pub trait Repo: Profile { let diff = surf::diff::Diff::try_from(diff)?; if highlight { - let mut hi = Highlighter::default(); + let mut hi = Highlighter::new(); return Ok::<_, Error>(diff.pretty(&mut hi, &(), &repo)); } diff --git a/public/syntax.css b/public/syntax.css index 0305995..0743ef7 100644 --- a/public/syntax.css +++ b/public/syntax.css @@ -1,4 +1,5 @@ .syntax.operator, +.syntax.keyword\.repeat, .syntax.keyword { color: var(--color-prettylights-syntax-keyword); } @@ -23,7 +24,8 @@ .syntax.function { color: var(--color-prettylights-syntax-constant); } -.syntax.comment { +.syntax.comment, +.syntax.comment\.documentation { color: var(--color-prettylights-syntax-comment); } .syntax.string { @@ -31,7 +33,8 @@ } .syntax.string.special { } -.syntax.function { +.syntax.function\.method { + color: var(--color-prettylights-syntax-entity); } .syntax.type.builtin { } @@ -57,4 +60,8 @@ .syntax.variable.parameter { } .syntax.constructor { + color: var(--color-prettylights-syntax-entity); +} +.syntax.tag\.delimiter { + color: var(--color-prettylights-syntax-keyword); }