From 8ac5b1d8037ab0883a34855e938adee93d7a1386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Mon, 12 Jun 2023 08:42:29 +0200 Subject: [PATCH 01/19] Initial commit for semantic tokenizer. --- src/all.cr | 2 + src/ast.cr | 5 +- src/ast/html_component.cr | 5 +- src/ast/html_element.cr | 2 + src/ast/node.cr | 12 +- src/cli.cr | 1 + src/commands/highlight.cr | 84 +++++++++++ src/parser.cr | 10 +- src/parsers/array_literal.cr | 2 +- src/parsers/case.cr | 4 +- src/parsers/component.cr | 4 +- src/parsers/connect.cr | 4 +- src/parsers/connect_variable.cr | 2 +- src/parsers/constant.cr | 2 +- src/parsers/css_font_face.cr | 2 +- src/parsers/css_keyframes.cr | 2 +- src/parsers/decode.cr | 6 +- src/parsers/directives/asset.cr | 2 +- src/parsers/directives/documentation.cr | 2 +- src/parsers/directives/format.cr | 2 +- src/parsers/directives/inline.cr | 2 +- src/parsers/directives/svg.cr | 2 +- src/parsers/encode.cr | 2 +- src/parsers/enum.cr | 2 +- src/parsers/for.cr | 4 +- src/parsers/for_condition.cr | 2 +- src/parsers/function.cr | 2 +- src/parsers/html_body.cr | 6 +- src/parsers/html_component.cr | 14 +- src/parsers/html_element.cr | 14 +- src/parsers/if.cr | 4 +- src/parsers/js.cr | 2 +- src/parsers/module.cr | 2 +- src/parsers/next_call.cr | 2 +- src/parsers/property.cr | 2 +- src/parsers/provider.cr | 2 +- src/parsers/record_definition.cr | 2 +- src/parsers/record_definition_field.cr | 2 +- src/parsers/return.cr | 2 +- src/parsers/routes.cr | 2 +- src/parsers/state.cr | 2 +- src/parsers/statement.cr | 4 +- src/parsers/store.cr | 2 +- src/parsers/style.cr | 2 +- src/parsers/suite.cr | 2 +- src/parsers/test.cr | 2 +- src/parsers/use.cr | 4 +- src/parsers/void.cr | 2 +- src/semantic_tokenizer.cr | 179 ++++++++++++++++++++++++ 49 files changed, 356 insertions(+), 70 deletions(-) create mode 100644 src/commands/highlight.cr create mode 100644 src/semantic_tokenizer.cr diff --git a/src/all.cr b/src/all.cr index de99cefb9..c9f6e48d1 100644 --- a/src/all.cr +++ b/src/all.cr @@ -57,6 +57,8 @@ require "./documentation_generator/**" require "./documentation_generator" require "./documentation_server" +require "./semantic_tokenizer" + require "./test_runner/**" require "./test_runner" diff --git a/src/ast.cr b/src/ast.cr index 9e123e314..beb686979 100644 --- a/src/ast.cr +++ b/src/ast.cr @@ -31,9 +31,10 @@ module Mint Js getter components, modules, records, stores, routes, providers - getter suites, enums, comments, nodes, unified_modules + getter suites, enums, comments, nodes, unified_modules, keywords - def initialize(@records = [] of RecordDefinition, + def initialize(@keywords = [] of Tuple(Int32, Int32), + @records = [] of RecordDefinition, @unified_modules = [] of Module, @components = [] of Component, @providers = [] of Provider, diff --git a/src/ast/html_component.cr b/src/ast/html_component.cr index 99fefbdff..f999db33b 100644 --- a/src/ast/html_component.cr +++ b/src/ast/html_component.cr @@ -1,13 +1,14 @@ module Mint class Ast class HtmlComponent < Node - getter attributes, children, component, comments, ref + getter attributes, children, component, comments, ref, closing_tag_position def initialize(@attributes : Array(HtmlAttribute), + @closing_tag_position : Int32?, @comments : Array(Comment), @children : Array(Node), - @ref : Variable?, @component : TypeId, + @ref : Variable?, @input : Data, @from : Int32, @to : Int32) diff --git a/src/ast/html_element.cr b/src/ast/html_element.cr index de4136589..d587353c6 100644 --- a/src/ast/html_element.cr +++ b/src/ast/html_element.cr @@ -2,8 +2,10 @@ module Mint class Ast class HtmlElement < Node getter attributes, children, styles, tag, comments, ref + getter closing_tag_position def initialize(@attributes : Array(HtmlAttribute), + @closing_tag_position : Int32?, @comments : Array(Comment), @styles : Array(HtmlStyle), @children : Array(Node), diff --git a/src/ast/node.cr b/src/ast/node.cr index 725b64d8f..9a1565fa6 100644 --- a/src/ast/node.cr +++ b/src/ast/node.cr @@ -71,7 +71,7 @@ module Mint source.strip.includes?('\n') end - protected def compute_position(lines, needle) : Position + def self.compute_position(lines, needle) : Position line_start_pos, line = begin left, right = 0, lines.size - 1 index = pos = 0 @@ -107,19 +107,23 @@ module Mint {line, column} end - getter location : Location do + def self.compute_location(input : Data, from, to) # TODO: avoid creating this array for every (initial) call to `Node#location` lines = [0] - @input.input.each_char_with_index do |ch, i| + input.input.each_char_with_index do |ch, i| lines << i + 1 if ch == '\n' end Location.new( - filename: @input.file, + filename: input.file, start: compute_position(lines, from), end: compute_position(lines, to), ) end + + getter location : Location do + Node.compute_location(input, from, to) + end end end end diff --git a/src/cli.cr b/src/cli.cr index 050e9f48f..dad26230e 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -15,6 +15,7 @@ module Mint define_help description: "Mint" register_sub_command "sandbox-server", type: SandboxServer + register_sub_command highlight, type: Highlight register_sub_command install, type: Install register_sub_command compile, type: Compile register_sub_command version, type: Version diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr new file mode 100644 index 000000000..2917777df --- /dev/null +++ b/src/commands/highlight.cr @@ -0,0 +1,84 @@ +module Mint + class Cli < Admiral::Command + class Highlight < Admiral::Command + include Command + + define_help description: "Returns the syntax highlighted version of the given file as HTML" + + define_argument path, + description: "The path to the file" + + def run + if path = arguments.path + ast = + Parser.parse(path) + + tokenizer = SemanticTokenizer.new + tokenizer.tokenize(ast) + + parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) + contents = File.read(path) + position = 0 + + tokenizer.tokens.sort_by(&.from).each do |token| + if token.from > position + parts << contents[position, token.from - position] + end + + parts << {contents[token.from, token.to - token.from], token.type} + position = token.to + end + + if position < contents.size + parts << contents[position, contents.size] + end + + result = parts.reduce("") do |memo, item| + memo + case item + in String + item + in Tuple(String, SemanticTokenizer::TokenType) + case item[1] + in SemanticTokenizer::TokenType::Type + item[0].colorize(:yellow) + in SemanticTokenizer::TokenType::TypeParameter + item[0].colorize(:light_yellow) + in SemanticTokenizer::TokenType::Variable + item[0].colorize(:dark_gray) + in SemanticTokenizer::TokenType::Class + item[0].colorize(:blue) + in SemanticTokenizer::TokenType::Struct + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::Namespace + item[0].colorize(:light_blue) + in SemanticTokenizer::TokenType::Function + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::Keyword + item[0].colorize(:magenta) + in SemanticTokenizer::TokenType::Property + item[0].colorize(:dark_gray).mode(:underline) + in SemanticTokenizer::TokenType::Comment + item[0].colorize(:light_gray) + in SemanticTokenizer::TokenType::Enum + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::EnumMember + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::String + item[0].colorize(:green) + in SemanticTokenizer::TokenType::Number + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::Regexp + item[0].colorize.fore(:white).back(:red) + in SemanticTokenizer::TokenType::Operator + item[0].colorize.fore(:white).back(:red) + end.to_s + # %(#{item[0]}) + end + end + + print result + end + end + end + end +end diff --git a/src/parser.cr b/src/parser.cr index 0f6da5c6c..fef8f06aa 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -167,8 +167,8 @@ module Mint # Consuming keywords # ---------------------------------------------------------------------------- - def keyword!(word, error) : Bool - keyword(word) || raise error + def keyword!(word, error, save : Bool = false) : Bool + keyword(word, save) || raise error end def keyword_ahead?(word) : Bool @@ -178,8 +178,12 @@ module Mint true end - def keyword(word) : Bool + def keyword(word, save : Bool = false) : Bool if keyword_ahead?(word) + if save + @ast.keywords << {position, position + word.size} + end + @position += word.size true else diff --git a/src/parsers/array_literal.cr b/src/parsers/array_literal.cr index 384c32ace..3d255b156 100644 --- a/src/parsers/array_literal.cr +++ b/src/parsers/array_literal.cr @@ -18,7 +18,7 @@ module Mint type = start do whitespace - next unless keyword "of" + next unless keyword("of", true) whitespace type_or_type_variable! ArrayLiteralExpectedTypeOrVariable end diff --git a/src/parsers/case.cr b/src/parsers/case.cr index 2914179e3..3c404b0db 100644 --- a/src/parsers/case.cr +++ b/src/parsers/case.cr @@ -8,14 +8,14 @@ module Mint def case_expression(for_css : Bool = false) : Ast::Case? start do |start_position| - next unless keyword "case" + next unless keyword("case", true) whitespace parens = char! '(' whitespace - await = keyword "await" + await = keyword("await", true) whitespace condition = expression! CaseExpectedCondition diff --git a/src/parsers/component.cr b/src/parsers/component.cr index 53b5bc289..37a3667ec 100644 --- a/src/parsers/component.cr +++ b/src/parsers/component.cr @@ -9,10 +9,10 @@ module Mint start do |start_position| comment = self.comment - global = keyword "global" + global = keyword("global", true) whitespace - next unless keyword "component" + next unless keyword("component", true) whitespace name = type_id! ComponentExpectedName diff --git a/src/parsers/connect.cr b/src/parsers/connect.cr index 5229a4062..cc0cdf312 100644 --- a/src/parsers/connect.cr +++ b/src/parsers/connect.cr @@ -8,13 +8,13 @@ module Mint def connect : Ast::Connect? start do |start_position| - next unless keyword "connect" + next unless keyword("connect", true) whitespace store = type_id! ConnectExpectedType whitespace - keyword! "exposing", ConnectExpectedExposing + keyword!("exposing", ConnectExpectedExposing, true) keys = block( opening_bracket: ConnectExpectedOpeningBracket, diff --git a/src/parsers/connect_variable.cr b/src/parsers/connect_variable.cr index 62f195174..cd5b10cc3 100644 --- a/src/parsers/connect_variable.cr +++ b/src/parsers/connect_variable.cr @@ -10,7 +10,7 @@ module Mint whitespace - if keyword "as" + if keyword("as", true) whitespace name = variable! ConnectVariableExpectedAs end diff --git a/src/parsers/constant.cr b/src/parsers/constant.cr index afe96bd84..af5396c91 100644 --- a/src/parsers/constant.cr +++ b/src/parsers/constant.cr @@ -9,7 +9,7 @@ module Mint comment = self.comment whitespace - next unless keyword "const" + next unless keyword("const", true) whitespace name = variable_constant! diff --git a/src/parsers/css_font_face.cr b/src/parsers/css_font_face.cr index ecbd9493f..48775888a 100644 --- a/src/parsers/css_font_face.cr +++ b/src/parsers/css_font_face.cr @@ -5,7 +5,7 @@ module Mint def css_font_face : Ast::CssFontFace? start do |start_position| - next unless keyword "@font-face" + next unless keyword("@font-face", true) definitions = block( opening_bracket: CssFontFaceExpectedOpeningBracket, diff --git a/src/parsers/css_keyframes.cr b/src/parsers/css_keyframes.cr index ddb157c5c..d3a725d59 100644 --- a/src/parsers/css_keyframes.cr +++ b/src/parsers/css_keyframes.cr @@ -6,7 +6,7 @@ module Mint def css_keyframes : Ast::CssKeyframes? start do |start_position| - next unless keyword "@keyframes" + next unless keyword("@keyframes", true) whitespace diff --git a/src/parsers/decode.cr b/src/parsers/decode.cr index 473491522..564f27f35 100644 --- a/src/parsers/decode.cr +++ b/src/parsers/decode.cr @@ -6,15 +6,15 @@ module Mint def decode : Ast::Decode? start do |start_position| - next unless keyword "decode" + next unless keyword("decode", true) next unless whitespace? whitespace - unless keyword "as" + unless keyword("as", true) expression = expression! DecodeExpectedExpression whitespace - keyword! "as", DecodeExpectedAs + keyword!("as", DecodeExpectedAs, true) end whitespace diff --git a/src/parsers/directives/asset.cr b/src/parsers/directives/asset.cr index a08f51e00..ed6652695 100644 --- a/src/parsers/directives/asset.cr +++ b/src/parsers/directives/asset.cr @@ -6,7 +6,7 @@ module Mint def asset_directive : Ast::Directives::Asset? start do |start_position| - next unless keyword "@asset" + next unless keyword("@asset", true) char '(', AssetDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/documentation.cr b/src/parsers/directives/documentation.cr index 3d36434d3..abcc8118f 100644 --- a/src/parsers/directives/documentation.cr +++ b/src/parsers/directives/documentation.cr @@ -6,7 +6,7 @@ module Mint def documentation_directive : Ast::Directives::Documentation? start do |start_position| - next unless keyword "@documentation" + next unless keyword("@documentation", true) char '(', DocumentationDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/format.cr b/src/parsers/directives/format.cr index f16a07470..7cf7856f0 100644 --- a/src/parsers/directives/format.cr +++ b/src/parsers/directives/format.cr @@ -6,7 +6,7 @@ module Mint def format_directive : Ast::Directives::Format? start do |start_position| - next unless keyword "@format" + next unless keyword("@format", true) content = code_block( diff --git a/src/parsers/directives/inline.cr b/src/parsers/directives/inline.cr index 600bc4abc..ca84342fd 100644 --- a/src/parsers/directives/inline.cr +++ b/src/parsers/directives/inline.cr @@ -6,7 +6,7 @@ module Mint def inline_directive : Ast::Directives::Inline? start do |start_position| - next unless keyword "@inline" + next unless keyword("@inline", true) char '(', InlineDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/svg.cr b/src/parsers/directives/svg.cr index 75b9b4acf..ab62cbde3 100644 --- a/src/parsers/directives/svg.cr +++ b/src/parsers/directives/svg.cr @@ -6,7 +6,7 @@ module Mint def svg_directive : Ast::Directives::Svg? start do |start_position| - next unless keyword "@svg" + next unless keyword("@svg", true) char '(', SvgDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/encode.cr b/src/parsers/encode.cr index 665d8fbd1..634ea89d0 100644 --- a/src/parsers/encode.cr +++ b/src/parsers/encode.cr @@ -4,7 +4,7 @@ module Mint def encode : Ast::Encode? start do |start_position| - next unless keyword "encode" + next unless keyword("encode", true) next unless whitespace? whitespace diff --git a/src/parsers/enum.cr b/src/parsers/enum.cr index 1a54106d3..50a03279e 100644 --- a/src/parsers/enum.cr +++ b/src/parsers/enum.cr @@ -9,7 +9,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword "enum" + next unless keyword("enum", true) whitespace name = type_id! EnumExpectedName diff --git a/src/parsers/for.cr b/src/parsers/for.cr index 34ba64a49..54886f668 100644 --- a/src/parsers/for.cr +++ b/src/parsers/for.cr @@ -9,7 +9,7 @@ module Mint def for_expression : Ast::For? start do |start_position| - next unless keyword "for" + next unless keyword("for", true) next unless whitespace? whitespace @@ -22,7 +22,7 @@ module Mint ) { variable } whitespace - keyword! "of", ForExpectedOf + keyword!("of", ForExpectedOf, true) whitespace subject = expression! ForExpectedSubject diff --git a/src/parsers/for_condition.cr b/src/parsers/for_condition.cr index 2b48d7db8..cae526100 100644 --- a/src/parsers/for_condition.cr +++ b/src/parsers/for_condition.cr @@ -6,7 +6,7 @@ module Mint def for_condition : Ast::ForCondition? start do |start_position| - next unless keyword "when" + next unless keyword("when", true) whitespace condition = diff --git a/src/parsers/function.cr b/src/parsers/function.cr index d1ab46238..519aedcc3 100644 --- a/src/parsers/function.cr +++ b/src/parsers/function.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword "fun" + next unless keyword("fun", true) whitespace name = variable! FunctionExpectedName, track: false diff --git a/src/parsers/html_body.cr b/src/parsers/html_body.cr index b9a99d3d6..85183c3dd 100644 --- a/src/parsers/html_body.cr +++ b/src/parsers/html_body.cr @@ -44,6 +44,9 @@ module Mint tag end + closing_tag_position = + position + 2 + raise expected_closing_tag, position, { "opening_tag" => tag, } unless keyword "" @@ -60,7 +63,8 @@ module Mint {attributes, children, - comments} + comments, + closing_tag_position} end end end diff --git a/src/parsers/html_component.cr b/src/parsers/html_component.cr index 88d32eeb7..b04851bee 100644 --- a/src/parsers/html_component.cr +++ b/src/parsers/html_component.cr @@ -16,18 +16,20 @@ module Mint ref = start do whitespace - next unless keyword "as" + next unless keyword("as", true) whitespace variable! HtmlComponentExpectedReference end - attributes, children, comments = html_body( - expected_closing_bracket: HtmlComponentExpectedClosingBracket, - expected_closing_tag: HtmlComponentExpectedClosingTag, - with_dashes: false, - tag: component) + attributes, children, comments, closing_tag_position = + html_body( + expected_closing_bracket: HtmlComponentExpectedClosingBracket, + expected_closing_tag: HtmlComponentExpectedClosingTag, + with_dashes: false, + tag: component) node = self << Ast::HtmlComponent.new( + closing_tag_position: closing_tag_position, attributes: attributes, from: start_position, component: component, diff --git a/src/parsers/html_element.cr b/src/parsers/html_element.cr index b9a888091..978932750 100644 --- a/src/parsers/html_element.cr +++ b/src/parsers/html_element.cr @@ -25,18 +25,20 @@ module Mint ref = start do whitespace - next unless keyword "as" + next unless keyword("as", true) whitespace variable! HtmlElementExpectedReference end - attributes, children, comments = html_body( - expected_closing_bracket: HtmlElementExpectedClosingBracket, - expected_closing_tag: HtmlElementExpectedClosingTag, - with_dashes: true, - tag: tag) + attributes, children, comments, closing_tag_position = + html_body( + expected_closing_bracket: HtmlElementExpectedClosingBracket, + expected_closing_tag: HtmlElementExpectedClosingTag, + with_dashes: true, + tag: tag) node = self << Ast::HtmlElement.new( + closing_tag_position: closing_tag_position, attributes: attributes, from: start_position, children: children, diff --git a/src/parsers/if.cr b/src/parsers/if.cr index 54f47a607..5659683d5 100644 --- a/src/parsers/if.cr +++ b/src/parsers/if.cr @@ -11,7 +11,7 @@ module Mint def if_expression(for_css = false) : Ast::If? start do |start_position| - next unless keyword "if" + next unless keyword("if", true) whitespace parens = char! '(' @@ -40,7 +40,7 @@ module Mint falsy = nil whitespace - if keyword "else" + if keyword("else", true) whitespace unless falsy = if_expression(for_css: for_css) diff --git a/src/parsers/js.cr b/src/parsers/js.cr index 90e5c2d32..b7edee0ac 100644 --- a/src/parsers/js.cr +++ b/src/parsers/js.cr @@ -15,7 +15,7 @@ module Mint type = start do whitespace - next unless keyword "as" + next unless keyword("as", true) whitespace type_or_type_variable! JsExpectedTypeOrVariable end diff --git a/src/parsers/module.cr b/src/parsers/module.cr index f49d8b4dc..2f82dc448 100644 --- a/src/parsers/module.cr +++ b/src/parsers/module.cr @@ -9,7 +9,7 @@ module Mint comment = self.comment whitespace - next unless keyword "module" + next unless keyword("module", true) whitespace name = type_id! ModuleExpectedName diff --git a/src/parsers/next_call.cr b/src/parsers/next_call.cr index 838995c7e..c42dd3595 100644 --- a/src/parsers/next_call.cr +++ b/src/parsers/next_call.cr @@ -4,7 +4,7 @@ module Mint def next_call : Ast::NextCall? start do |start_position| - next unless keyword "next" + next unless keyword("next", true) next unless whitespace? whitespace diff --git a/src/parsers/property.cr b/src/parsers/property.cr index 5e61b036f..6cdf33d03 100644 --- a/src/parsers/property.cr +++ b/src/parsers/property.cr @@ -11,7 +11,7 @@ module Mint start_position = position - next unless keyword "property" + next unless keyword("property", true) whitespace name = variable! PropertyExpectedName, track: false diff --git a/src/parsers/provider.cr b/src/parsers/provider.cr index 74709263e..f7e78b593 100644 --- a/src/parsers/provider.cr +++ b/src/parsers/provider.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword "provider" + next unless keyword("provider", true) whitespace name = type_id! ProviderExpectedName diff --git a/src/parsers/record_definition.cr b/src/parsers/record_definition.cr index 9da509ae4..309306658 100644 --- a/src/parsers/record_definition.cr +++ b/src/parsers/record_definition.cr @@ -8,7 +8,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword "record" + next unless keyword("record", true) whitespace name = type_id! RecordDefinitionExpectedName diff --git a/src/parsers/record_definition_field.cr b/src/parsers/record_definition_field.cr index e145bbf47..0f77eca02 100644 --- a/src/parsers/record_definition_field.cr +++ b/src/parsers/record_definition_field.cr @@ -19,7 +19,7 @@ module Mint mapping = start do whitespace - next unless keyword "using" + next unless keyword("using", true) whitespace string_literal! RecordDefinitionFieldExpectedMapping, with_interpolation: false diff --git a/src/parsers/return.cr b/src/parsers/return.cr index 342e967e7..ac40debfc 100644 --- a/src/parsers/return.cr +++ b/src/parsers/return.cr @@ -4,7 +4,7 @@ module Mint def return_call : Ast::ReturnCall? start do |start_position| - next unless keyword "return" + next unless keyword("return", true) next unless whitespace? whitespace diff --git a/src/parsers/routes.cr b/src/parsers/routes.cr index 1392b44a2..c6cbdb4e3 100644 --- a/src/parsers/routes.cr +++ b/src/parsers/routes.cr @@ -6,7 +6,7 @@ module Mint def routes : Ast::Routes? start do |start_position| - next unless keyword "routes" + next unless keyword("routes", true) body = block( opening_bracket: RoutesExpectedOpeningBracket, diff --git a/src/parsers/state.cr b/src/parsers/state.cr index 2518a84c9..12dfe60a8 100644 --- a/src/parsers/state.cr +++ b/src/parsers/state.cr @@ -10,7 +10,7 @@ module Mint comment = self.comment whitespace - next unless keyword "state" + next unless keyword("state", true) whitespace name = variable! StateExpectedName diff --git a/src/parsers/statement.cr b/src/parsers/statement.cr index 2cdc4d09a..bac128e5e 100644 --- a/src/parsers/statement.cr +++ b/src/parsers/statement.cr @@ -3,7 +3,7 @@ module Mint def statement : Ast::Statement? start do |start_position| target = start do - next unless keyword "let" + next unless keyword("let", true) whitespace value = variable(track: false) || @@ -19,7 +19,7 @@ module Mint end whitespace - await = keyword "await" + await = keyword("await", true) whitespace body = expression diff --git a/src/parsers/store.cr b/src/parsers/store.cr index 0f2f746e2..0963940ae 100644 --- a/src/parsers/store.cr +++ b/src/parsers/store.cr @@ -10,7 +10,7 @@ module Mint comment = self.comment whitespace - next unless keyword "store" + next unless keyword("store", true) whitespace name = type_id! StoreExpectedName diff --git a/src/parsers/style.cr b/src/parsers/style.cr index 50352f5d3..33b2bb13f 100644 --- a/src/parsers/style.cr +++ b/src/parsers/style.cr @@ -7,7 +7,7 @@ module Mint def style : Ast::Style? start do |start_position| - next unless keyword "style" + next unless keyword("style", true) whitespace name = variable_with_dashes! StyleExpectedName diff --git a/src/parsers/suite.cr b/src/parsers/suite.cr index c22336995..dac4514d1 100644 --- a/src/parsers/suite.cr +++ b/src/parsers/suite.cr @@ -7,7 +7,7 @@ module Mint def suite : Ast::Suite? start do |start_position| - next unless keyword "suite" + next unless keyword("suite", true) whitespace diff --git a/src/parsers/test.cr b/src/parsers/test.cr index 8765f569e..4f0b9d9c7 100644 --- a/src/parsers/test.cr +++ b/src/parsers/test.cr @@ -7,7 +7,7 @@ module Mint def test : Ast::Test? start do |start_position| - next unless keyword "test" + next unless keyword("test", true) whitespace diff --git a/src/parsers/use.cr b/src/parsers/use.cr index 1dad13e69..4454a978d 100644 --- a/src/parsers/use.cr +++ b/src/parsers/use.cr @@ -8,7 +8,7 @@ module Mint def use : Ast::Use? start do |start_position| - next unless keyword "use" + next unless keyword("use", true) whitespace provider = type_id! UseExpectedProvider @@ -17,7 +17,7 @@ module Mint raise UseExpectedRecord unless item = record whitespace - if keyword "when" + if keyword("when", true) condition = block( opening_bracket: UseExpectedOpeningBracket, closing_bracket: UseExpectedClosingBracket diff --git a/src/parsers/void.cr b/src/parsers/void.cr index f35238456..f44a51acf 100644 --- a/src/parsers/void.cr +++ b/src/parsers/void.cr @@ -2,7 +2,7 @@ module Mint class Parser def void : Ast::Void? start do |start_position| - next unless keyword "void" + next unless keyword("void", true) self << Ast::Void.new( from: start_position, diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr new file mode 100644 index 000000000..40c7d446d --- /dev/null +++ b/src/semantic_tokenizer.cr @@ -0,0 +1,179 @@ +module Mint + class SemanticTokenizer + enum TokenType + Type + TypeParameter + + Variable + + Class # Component + Struct # Record + Namespace # HTML Tags + Function + Keyword + Property + Comment + + Enum + EnumMember + + String + Number + Regexp + Operator + end + + record Token, + type : TokenType, + from : Int32, + to : Int32 + + getter tokens : Array(Token) = [] of Token + + def tokenize(ast : Ast) + ast.keywords.each do |(from, to)| + add(from, to, TokenType::Keyword) + end + + tokenize( + ast.records + + ast.providers + + ast.components + + ast.modules + + ast.routes + + ast.stores + + ast.suites + + ast.enums + + ast.comments) + end + + def tokenize(node : Ast::Component) + add(node.name, TokenType::Type) + + tokenize(node.comment) + + tokenize( + node.properties + + node.functions + + node.constants + + node.connects + + node.comments + + node.styles + + node.states + + node.gets + + node.uses) + end + + def tokenize(node : Ast::Style) + add(node.name, TokenType::Variable) + tokenize(node.arguments) + tokenize(node.body) + end + + def tokenize(node : Ast::CssDefinition) + add(node.from, node.from + node.name.size, TokenType::Property) + tokenize(node.value.select(Ast::Node)) + end + + def tokenize(node : Ast::Function) + add(node.name, TokenType::Variable) + + tokenize(node.arguments) + tokenize(node.comment) + tokenize(node.type) + tokenize(node.body) + end + + def tokenize(node : Ast::Type) + add(node.name, TokenType::Type) + + tokenize(node.parameters) + end + + def tokenize(node : Ast::Block) + tokenize(node.statements) + end + + def tokenize(node : Ast::HtmlElement) + node.closing_tag_position.try do |position| + add(position, position + node.tag.value.size, TokenType::Namespace) + end + + add(node.tag, TokenType::Namespace) + add(node.ref, TokenType::Variable) + + tokenize(node.attributes) + tokenize(node.comments) + tokenize(node.children) + tokenize(node.styles) + end + + def tokenize(node : Ast::HtmlComponent) + node.closing_tag_position.try do |position| + add(position, position + node.component.value.size, TokenType::Type) + end + + add(node.component, TokenType::Type) + add(node.ref, TokenType::Variable) + + tokenize(node.attributes) + tokenize(node.comments) + tokenize(node.children) + end + + def tokenize(node : Ast::HtmlAttribute) + add(node.name, TokenType::Variable) + + tokenize(node.value) + end + + def tokenize(node : Ast::HtmlStyle) + add(node.name, TokenType::Variable) + + tokenize(node.arguments) + end + + def tokenize(node : Ast::StringLiteral) + add(node, TokenType::String) + end + + def tokenize(node : Ast::Statement) + tokenize(node.expression) + tokenize(node.target) + end + + def tokenize(node : Ast::Comment) + add(node, TokenType::Comment) + end + + def add(from : Int32, to : Int32, type : TokenType) + tokens << Token.new( + type: type, + from: from, + to: to) + end + + def add(node : Ast::Node | Nil, type : TokenType) + add(node.from, node.to, type) if node + end + + def tokenize(nodes : Array(Ast::Node)) + nodes.each { |node| tokenize(node) } + end + + def tokenize(node : Nil) + end + + def tokenize(node : Ast::Node) + puts "Tokenizer is not implemented for: #{node.class}" + end + end +end + +{% for subclass in Mint::Ast::Node.subclasses %} + {% puts "Missing implementation of tokenize for #{subclass.name}" unless Mint::SemanticTokenizer.methods.any? do |method| + if method.name == "tokenize" + method.args[0].restriction.id == subclass.name.gsub(/Mint::/, "") + end + end %} +{% end %} From 60f1bb51614ca1074afcce15052e1cdb84314fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 13 Jun 2023 10:46:23 +0200 Subject: [PATCH 02/19] Simplify semantic tokenizer a bit. --- src/commands/highlight.cr | 2 +- src/ls/definition/type_id.cr | 30 ++++++++++ src/parser.cr | 6 ++ src/parsers/get.cr | 2 +- src/parsers/type_id.cr | 20 ++++--- src/semantic_tokenizer.cr | 111 +++++++---------------------------- 6 files changed, 70 insertions(+), 101 deletions(-) create mode 100644 src/ls/definition/type_id.cr diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr index 2917777df..b60419da4 100644 --- a/src/commands/highlight.cr +++ b/src/commands/highlight.cr @@ -66,7 +66,7 @@ module Mint in SemanticTokenizer::TokenType::String item[0].colorize(:green) in SemanticTokenizer::TokenType::Number - item[0].colorize.fore(:white).back(:red) + item[0].colorize(:red) in SemanticTokenizer::TokenType::Regexp item[0].colorize.fore(:white).back(:red) in SemanticTokenizer::TokenType::Operator diff --git a/src/ls/definition/type_id.cr b/src/ls/definition/type_id.cr new file mode 100644 index 000000000..f8b15a051 --- /dev/null +++ b/src/ls/definition/type_id.cr @@ -0,0 +1,30 @@ +module Mint + module LS + class Definition < LSP::RequestMessage + def definition(node : Ast::TypeId, server : Server, workspace : Workspace, stack : Array(Ast::Node)) + found = + workspace.ast.enums.find(&.name.value.==(node.value)) || + workspace.ast.records.find(&.name.value.==(node.value)) || + workspace.ast.stores.find(&.name.value.==(node.value)) || + find_component(workspace, node.value) + + if found.nil? && (next_node = stack[1]) + definition(next_node, server, workspace, stack) + else + return if Core.ast.nodes.includes?(found) + + case found + when Ast::Store + location_link server, node, found.name, found + when Ast::Enum + location_link server, node, found.name, found + when Ast::Component + location_link server, node, found.name, found + when Ast::RecordDefinition + location_link server, node, found.name, found + end + end + end + end + end +end diff --git a/src/parser.cr b/src/parser.cr index fef8f06aa..adbc3237d 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -26,13 +26,19 @@ module Mint begin node = yield position @position = start_position unless node + clear_nodes(start_position) unless node node rescue error : Error @position = start_position + clear_nodes(start_position) raise error end end + def clear_nodes(from_position) + ast.nodes.reject! { |node| node.from >= from_position } + end + def step @position += 1 end diff --git a/src/parsers/get.cr b/src/parsers/get.cr index 54593ab7b..a61b9f29a 100644 --- a/src/parsers/get.cr +++ b/src/parsers/get.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword "get" + next unless keyword("get", true) whitespace name = variable! GetExpectedName, track: false diff --git a/src/parsers/type_id.cr b/src/parsers/type_id.cr index 56f55916a..aefea45d7 100644 --- a/src/parsers/type_id.cr +++ b/src/parsers/type_id.cr @@ -1,6 +1,6 @@ module Mint class Parser - def type_id!(error : SyntaxError.class) : Ast::TypeId + def type_id!(error : SyntaxError.class, track : Bool = true) : Ast::TypeId start do |start_position| value = gather do char(error, &.ascii_uppercase?) @@ -10,7 +10,7 @@ module Mint raise error unless value if char! '.' - other = type_id! error + other = type_id!(error, false) value += ".#{other.value}" end @@ -18,11 +18,13 @@ module Mint from: start_position, value: value, to: position, - input: data) + input: data).tap do |node| + self << node if track + end end end - def type_id : Ast::TypeId? + def type_id(track : Bool = true) : Ast::TypeId? start do |start_position| value = gather do return unless char.ascii_uppercase? @@ -36,7 +38,7 @@ module Mint if char == '.' other = start do step - next_part = type_id + next_part = type_id(false) next unless next_part next_part end @@ -51,13 +53,15 @@ module Mint from: start_position, value: value, to: position, - input: data) + input: data).tap do |node| + self << node if track + end end end - def type_id(error : SyntaxError.class) : Ast::TypeId? + def type_id(error : SyntaxError.class, track : Bool = true) : Ast::TypeId? return unless char.ascii_uppercase? - type_id! error + type_id! error, track end end end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index 40c7d446d..c6034ecff 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -35,63 +35,18 @@ module Mint add(from, to, TokenType::Keyword) end - tokenize( - ast.records + - ast.providers + - ast.components + - ast.modules + - ast.routes + - ast.stores + - ast.suites + - ast.enums + - ast.comments) - end - - def tokenize(node : Ast::Component) - add(node.name, TokenType::Type) - - tokenize(node.comment) - - tokenize( - node.properties + - node.functions + - node.constants + - node.connects + - node.comments + - node.styles + - node.states + - node.gets + - node.uses) - end - - def tokenize(node : Ast::Style) - add(node.name, TokenType::Variable) - tokenize(node.arguments) - tokenize(node.body) + tokenize(ast.nodes) end def tokenize(node : Ast::CssDefinition) add(node.from, node.from + node.name.size, TokenType::Property) - tokenize(node.value.select(Ast::Node)) end - def tokenize(node : Ast::Function) - add(node.name, TokenType::Variable) - - tokenize(node.arguments) - tokenize(node.comment) - tokenize(node.type) - tokenize(node.body) - end - - def tokenize(node : Ast::Type) - add(node.name, TokenType::Type) - - tokenize(node.parameters) - end - - def tokenize(node : Ast::Block) - tokenize(node.statements) + def tokenize(node : Ast::ArrayAccess) + case index = node.index + when Int64 + add(node.from + 1, node.from + 1 + index.to_s.size, TokenType::Number) + end end def tokenize(node : Ast::HtmlElement) @@ -100,50 +55,36 @@ module Mint end add(node.tag, TokenType::Namespace) - add(node.ref, TokenType::Variable) - - tokenize(node.attributes) - tokenize(node.comments) - tokenize(node.children) - tokenize(node.styles) end def tokenize(node : Ast::HtmlComponent) node.closing_tag_position.try do |position| add(position, position + node.component.value.size, TokenType::Type) end - - add(node.component, TokenType::Type) - add(node.ref, TokenType::Variable) - - tokenize(node.attributes) - tokenize(node.comments) - tokenize(node.children) end - def tokenize(node : Ast::HtmlAttribute) - add(node.name, TokenType::Variable) - - tokenize(node.value) + def tokenize(node : Ast::StringLiteral) + add(node, TokenType::String) end - def tokenize(node : Ast::HtmlStyle) - add(node.name, TokenType::Variable) + def tokenize(node : Ast::BoolLiteral) + add(node, TokenType::Keyword) + end - tokenize(node.arguments) + def tokenize(node : Ast::NumberLiteral) + add(node, TokenType::Number) end - def tokenize(node : Ast::StringLiteral) - add(node, TokenType::String) + def tokenize(node : Ast::Comment) + add(node, TokenType::Comment) end - def tokenize(node : Ast::Statement) - tokenize(node.expression) - tokenize(node.target) + def tokenize(node : Ast::Variable) + add(node, TokenType::Variable) end - def tokenize(node : Ast::Comment) - add(node, TokenType::Comment) + def tokenize(node : Ast::TypeId) + add(node, TokenType::Type) end def add(from : Int32, to : Int32, type : TokenType) @@ -161,19 +102,7 @@ module Mint nodes.each { |node| tokenize(node) } end - def tokenize(node : Nil) - end - - def tokenize(node : Ast::Node) - puts "Tokenizer is not implemented for: #{node.class}" + def tokenize(node : Ast::Node?) end end end - -{% for subclass in Mint::Ast::Node.subclasses %} - {% puts "Missing implementation of tokenize for #{subclass.name}" unless Mint::SemanticTokenizer.methods.any? do |method| - if method.name == "tokenize" - method.args[0].restriction.id == subclass.name.gsub(/Mint::/, "") - end - end %} -{% end %} From 0e865c680c836474022ebb05d28d49e6bf3f0b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 13 Jun 2023 15:56:57 +0200 Subject: [PATCH 03/19] Working language server implementation. --- spec/language_server/ls_spec.cr | 2 +- spec/language_server/semantic_tokens/record | 87 +++++++++++++++++++ src/ls/initialize.cr | 10 +++ src/ls/semantic_tokens.cr | 64 ++++++++++++++ src/ls/server.cr | 3 +- .../semantic_tokens_client_capabilities.cr | 16 ++++ src/lsp/protocol/semantic_tokens_options.cr | 33 +++++++ src/lsp/protocol/semantic_tokens_params.cr | 9 ++ src/lsp/protocol/server_capabilities.cr | 5 ++ .../text_document_client_capabilities.cr | 4 + src/parsers/comment.cr | 2 +- 11 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 spec/language_server/semantic_tokens/record create mode 100644 src/ls/semantic_tokens.cr create mode 100644 src/lsp/protocol/semantic_tokens_client_capabilities.cr create mode 100644 src/lsp/protocol/semantic_tokens_options.cr create mode 100644 src/lsp/protocol/semantic_tokens_params.cr diff --git a/spec/language_server/ls_spec.cr b/spec/language_server/ls_spec.cr index 78fd357a1..ef9b05518 100644 --- a/spec/language_server/ls_spec.cr +++ b/spec/language_server/ls_spec.cr @@ -5,7 +5,7 @@ def clean_json(workspace : Workspace, path : String) end Dir - .glob("./spec/language_server/{definition,hover}/**/*") + .glob("./spec/language_server/{definition,hover,semantic_tokens}/**/*") .select! { |file| File.file?(file) } .sort! .each do |file| diff --git a/spec/language_server/semantic_tokens/record b/spec/language_server/semantic_tokens/record new file mode 100644 index 000000000..a2b194b01 --- /dev/null +++ b/spec/language_server/semantic_tokens/record @@ -0,0 +1,87 @@ +/* Some comment. */ +record Article { + id : Number, + description : String, + title : String +} +-----------------------------------------------------------------file test.mint +{ + "id": 0, + "method": "initialize", + "params": { + "capabilities": { + "textDocument": { + "semanticTokens": { + "dynamicRegistration": false, + "tokenTypes": ["property"] + } + } + } + } +} +-------------------------------------------------------------------------request +{ + "jsonrpc": "2.0", + "id": 1, + "params": { + "textDocument": { + "uri": "file://#{root_path}/test.mint" + } + }, + "method": "textDocument/semanticTokens/full" +} +-------------------------------------------------------------------------request +{ + "jsonrpc": "2.0", + "result": { + "data": [ + 0, + 0, + 20, + 2, + 0, + 1, + 0, + 6, + 1, + 0, + 0, + 7, + 7, + 3, + 0, + 1, + 2, + 2, + 7, + 0, + 0, + 5, + 6, + 3, + 0, + 1, + 2, + 11, + 7, + 0, + 0, + 14, + 6, + 3, + 0, + 1, + 2, + 5, + 7, + 0, + 0, + 8, + 6, + 3, + 0 + ] + }, + "id": 1 +} +------------------------------------------------------------------------response diff --git a/src/ls/initialize.cr b/src/ls/initialize.cr index 0d7305fff..4064d11b1 100644 --- a/src/ls/initialize.cr +++ b/src/ls/initialize.cr @@ -42,10 +42,20 @@ module Mint change_notifications: false, supported: false)) + semantic_tokens_options = + LSP::SemanticTokensOptions.new( + range: false, + full: true, + legend: LSP::SemanticTokensLegend.new( + token_modifiers: [] of String, + token_types: ["class", "keyword", "comment", "type", "property", "number", "namespace", "variable", "string"] of String, + )) + capabilities = LSP::ServerCapabilities.new( document_on_type_formatting_provider: document_on_type_formatting_provider, execute_command_provider: execute_command_provider, + semantic_tokens_provider: semantic_tokens_options, signature_help_provider: signature_help_provider, document_link_provider: document_link_provider, completion_provider: completion_provider, diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr new file mode 100644 index 000000000..809508370 --- /dev/null +++ b/src/ls/semantic_tokens.cr @@ -0,0 +1,64 @@ +module Mint + module LS + # This is the class that handles the "textDocument/hover" request. + class SemanticTokens < LSP::RequestMessage + property params : LSP::SemanticTokensParams + + def execute(server) + uri = + URI.parse(params.text_document.uri) + + ast = + Parser.parse(uri.path.to_s) + + tokenizer = SemanticTokenizer.new + tokenizer.tokenize(ast) + + data = + tokenizer.tokens.sort_by(&.from).compact_map do |token| + input = + ast.nodes.first.input + + location = + Ast::Node.compute_location(input, token.from, token.to) + + type = + token.type.to_s.underscore + + if index = ["class", "keyword", "comment", "type", "property", "number", "namespace", "variable", "string"].index(type) + [ + location.start[0] - 1, + location.start[1], + token.to - token.from, + index, + 0, + ] + end + end + + result = [] of Array(Int32) + + data.each_with_index do |item, index| + current = + item.dup + + unless index.zero? + last = + data[index - 1] + + current[0] = + current[0] - last[0] + + current[1] = current[1] - last[1] if current[0] == 0 + end + + result << current + end + + { + data: result.flatten, + } + end + end + end +end diff --git a/src/ls/server.cr b/src/ls/server.cr index 98d3ba3c1..9166feae6 100644 --- a/src/ls/server.cr +++ b/src/ls/server.cr @@ -8,13 +8,14 @@ module Mint # Text document related methods method "textDocument/willSaveWaitUntil", WillSaveWaitUntil + method "textDocument/semanticTokens/full", SemanticTokens method "textDocument/foldingRange", FoldingRange method "textDocument/formatting", Formatting method "textDocument/completion", Completion method "textDocument/codeAction", CodeAction + method "textDocument/definition", Definition method "textDocument/didChange", DidChange method "textDocument/hover", Hover - method "textDocument/definition", Definition property params : LSP::InitializeParams? = nil diff --git a/src/lsp/protocol/semantic_tokens_client_capabilities.cr b/src/lsp/protocol/semantic_tokens_client_capabilities.cr new file mode 100644 index 000000000..a86d46ec2 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_client_capabilities.cr @@ -0,0 +1,16 @@ +module LSP + struct SemanticTokensClientCapabilities + include JSON::Serializable + + # Whether definition supports dynamic registration. + @[JSON::Field(key: "dynamicRegistration")] + property dynamic_registration : Bool? + + # The token types that the client supports. + @[JSON::Field(key: "tokenTypes")] + property token_types : Array(String) + + def initialize(@dynamic_registration = nil, @token_types = nil) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_options.cr b/src/lsp/protocol/semantic_tokens_options.cr new file mode 100644 index 000000000..425f112d8 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_options.cr @@ -0,0 +1,33 @@ +module LSP + struct SemanticTokensLegend + include JSON::Serializable + + # The token types a server uses. + @[JSON::Field(key: "tokenTypes")] + property token_types : Array(String) + + # The token modifiers a server uses. + @[JSON::Field(key: "tokenModifiers")] + property token_modifiers : Array(String) + + def initialize(@token_types, @token_modifiers) + end + end + + struct SemanticTokensOptions + include JSON::Serializable + + # The legend used by the server + property legend : SemanticTokensLegend + + # Server supports providing semantic tokens for a specific range + # of a document. + property? range : Bool + + # Server supports providing semantic tokens for a full document. + property? full : Bool + + def initialize(@legend, @range, @full) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_params.cr b/src/lsp/protocol/semantic_tokens_params.cr new file mode 100644 index 000000000..175b89d29 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_params.cr @@ -0,0 +1,9 @@ +module LSP + class SemanticTokensParams + include JSON::Serializable + + # The document in which the command was invoked. + @[JSON::Field(key: "textDocument")] + property text_document : TextDocumentIdentifier + end +end diff --git a/src/lsp/protocol/server_capabilities.cr b/src/lsp/protocol/server_capabilities.cr index b67cbef19..d3d1e18ca 100644 --- a/src/lsp/protocol/server_capabilities.cr +++ b/src/lsp/protocol/server_capabilities.cr @@ -96,6 +96,10 @@ module LSP @[JSON::Field(key: "executeCommandProvider")] property execute_command_provider : ExecuteCommandOptions + # The server provides semantic tokens support. + @[JSON::Field(key: "semanticTokensProvider")] + property semantic_tokens_provider : SemanticTokensOptions + # Workspace specific server capabilities property workspace : Workspace @@ -105,6 +109,7 @@ module LSP @document_formatting_provider, @document_highlight_provider, @workspace_symbol_provider, + @semantic_tokens_provider, @document_symbol_provider, @type_definition_provider, @execute_command_provider, diff --git a/src/lsp/protocol/text_document_client_capabilities.cr b/src/lsp/protocol/text_document_client_capabilities.cr index b2a8af71e..8aab5e82f 100644 --- a/src/lsp/protocol/text_document_client_capabilities.cr +++ b/src/lsp/protocol/text_document_client_capabilities.cr @@ -7,5 +7,9 @@ module LSP # Capabilities specific to the `textDocument/definition` request. property definition : DefinitionClientCapabilities? + + # Capabilities specific to the `textDocument/semanticTokens` request. + @[JSON::Field(key: "semanticTokens")] + property semantic_tokens : SemanticTokensClientCapabilities? end end diff --git a/src/parsers/comment.cr b/src/parsers/comment.cr index 44b4af94c..2ddad63b0 100644 --- a/src/parsers/comment.cr +++ b/src/parsers/comment.cr @@ -24,7 +24,7 @@ module Mint whitespace - Ast::Comment.new( + self << Ast::Comment.new( from: start_position, value: value, type: type, From 1c14a5ae2fdca2764298be576db51f656d2a0126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 13 Jun 2023 16:23:09 +0200 Subject: [PATCH 04/19] Cleanup semantic tokenizer class. --- src/ast.cr | 4 +- src/commands/highlight.cr | 10 --- src/ls/semantic_tokens.cr | 6 +- src/lsp/protocol/semantic_tokens_legend.cr | 16 ++++ src/lsp/protocol/semantic_tokens_options.cr | 15 ---- src/parsers/operation.cr | 2 + src/semantic_tokenizer.cr | 91 ++++++++++----------- 7 files changed, 67 insertions(+), 77 deletions(-) create mode 100644 src/lsp/protocol/semantic_tokens_legend.cr diff --git a/src/ast.cr b/src/ast.cr index beb686979..539e32c83 100644 --- a/src/ast.cr +++ b/src/ast.cr @@ -32,8 +32,10 @@ module Mint getter components, modules, records, stores, routes, providers getter suites, enums, comments, nodes, unified_modules, keywords + getter operators - def initialize(@keywords = [] of Tuple(Int32, Int32), + def initialize(@operators = [] of Tuple(Int32, Int32), + @keywords = [] of Tuple(Int32, Int32), @records = [] of RecordDefinition, @unified_modules = [] of Module, @components = [] of Component, diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr index b60419da4..3950de5df 100644 --- a/src/commands/highlight.cr +++ b/src/commands/highlight.cr @@ -45,24 +45,14 @@ module Mint item[0].colorize(:light_yellow) in SemanticTokenizer::TokenType::Variable item[0].colorize(:dark_gray) - in SemanticTokenizer::TokenType::Class - item[0].colorize(:blue) - in SemanticTokenizer::TokenType::Struct - item[0].colorize.fore(:white).back(:red) in SemanticTokenizer::TokenType::Namespace item[0].colorize(:light_blue) - in SemanticTokenizer::TokenType::Function - item[0].colorize.fore(:white).back(:red) in SemanticTokenizer::TokenType::Keyword item[0].colorize(:magenta) in SemanticTokenizer::TokenType::Property item[0].colorize(:dark_gray).mode(:underline) in SemanticTokenizer::TokenType::Comment item[0].colorize(:light_gray) - in SemanticTokenizer::TokenType::Enum - item[0].colorize.fore(:white).back(:red) - in SemanticTokenizer::TokenType::EnumMember - item[0].colorize.fore(:white).back(:red) in SemanticTokenizer::TokenType::String item[0].colorize(:green) in SemanticTokenizer::TokenType::Number diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr index 809508370..ef299f27a 100644 --- a/src/ls/semantic_tokens.cr +++ b/src/ls/semantic_tokens.cr @@ -11,14 +11,14 @@ module Mint ast = Parser.parse(uri.path.to_s) + # This is used later on to convert the line/column of each token + input = + ast.nodes.first.input tokenizer = SemanticTokenizer.new tokenizer.tokenize(ast) data = tokenizer.tokens.sort_by(&.from).compact_map do |token| - input = - ast.nodes.first.input - location = Ast::Node.compute_location(input, token.from, token.to) diff --git a/src/lsp/protocol/semantic_tokens_legend.cr b/src/lsp/protocol/semantic_tokens_legend.cr new file mode 100644 index 000000000..55de94a39 --- /dev/null +++ b/src/lsp/protocol/semantic_tokens_legend.cr @@ -0,0 +1,16 @@ +module LSP + struct SemanticTokensLegend + include JSON::Serializable + + # The token types a server uses. + @[JSON::Field(key: "tokenTypes")] + property token_types : Array(String) + + # The token modifiers a server uses. + @[JSON::Field(key: "tokenModifiers")] + property token_modifiers : Array(String) + + def initialize(@token_types, @token_modifiers) + end + end +end diff --git a/src/lsp/protocol/semantic_tokens_options.cr b/src/lsp/protocol/semantic_tokens_options.cr index 425f112d8..49b2dd1b0 100644 --- a/src/lsp/protocol/semantic_tokens_options.cr +++ b/src/lsp/protocol/semantic_tokens_options.cr @@ -1,19 +1,4 @@ module LSP - struct SemanticTokensLegend - include JSON::Serializable - - # The token types a server uses. - @[JSON::Field(key: "tokenTypes")] - property token_types : Array(String) - - # The token modifiers a server uses. - @[JSON::Field(key: "tokenModifiers")] - property token_modifiers : Array(String) - - def initialize(@token_types, @token_modifiers) - end - end - struct SemanticTokensOptions include JSON::Serializable diff --git a/src/parsers/operation.cr b/src/parsers/operation.cr index 1ee471b8b..249db7cd0 100644 --- a/src/parsers/operation.cr +++ b/src/parsers/operation.cr @@ -30,9 +30,11 @@ module Mint def operator : String? start do whitespace + saved_position = position operator = OPERATORS.keys.find { |item| keyword item } next unless operator next unless whitespace? + ast.operators << {saved_position, saved_position + operator.size} whitespace operator end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index c6034ecff..aadd96e05 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -1,48 +1,71 @@ module Mint class SemanticTokenizer + # This is a subset of the LSPs SemanticTokenTypes enum. enum TokenType - Type TypeParameter + Type - Variable - - Class # Component - Struct # Record - Namespace # HTML Tags - Function - Keyword + Namespace Property + Keyword Comment - Enum - EnumMember - + Variable + Operator String Number Regexp - Operator end + # This represents which token types are used for which node. + TOKEN_MAP = { + Ast::TypeVariable => TokenType::TypeParameter, + Ast::Variable => TokenType::Variable, + Ast::BoolLiteral => TokenType::Keyword, + Ast::Comment => TokenType::Comment, + Ast::StringLiteral => TokenType::String, + Ast::NumberLiteral => TokenType::Number, + Ast::TypeId => TokenType::Type, + } + + # Represents a semantic token using the positions of the token instead + # of line / column (for the LSP it is converted to line /column). record Token, type : TokenType, from : Int32, to : Int32 + # We keep a cache of all tokenized nodes to avoid duplications + getter cache : Set(Ast::Node) = Set(Ast::Node).new + + # This is where the resulting tokens are stored. getter tokens : Array(Token) = [] of Token def tokenize(ast : Ast) - ast.keywords.each do |(from, to)| - add(from, to, TokenType::Keyword) - end + # We add the operators and keywords directly from the AST + ast.operators.each { |(from, to)| add(from, to, TokenType::Operator) } + ast.keywords.each { |(from, to)| add(from, to, TokenType::Keyword) } tokenize(ast.nodes) end + def tokenize(nodes : Array(Ast::Node)) + nodes.each { |node| tokenize(node) } + end + + def tokenize(node : Ast::Node?) + if type = TOKEN_MAP[node.class]? + add(node, type) + end + end + def tokenize(node : Ast::CssDefinition) add(node.from, node.from + node.name.size, TokenType::Property) end def tokenize(node : Ast::ArrayAccess) + # TODO: The index should be parsed as a number literal when + # implemented remove this case index = node.index when Int64 add(node.from + 1, node.from + 1 + index.to_s.size, TokenType::Number) @@ -50,6 +73,7 @@ module Mint end def tokenize(node : Ast::HtmlElement) + # The closing tag is not saved only the position to it. node.closing_tag_position.try do |position| add(position, position + node.tag.value.size, TokenType::Namespace) end @@ -63,30 +87,6 @@ module Mint end end - def tokenize(node : Ast::StringLiteral) - add(node, TokenType::String) - end - - def tokenize(node : Ast::BoolLiteral) - add(node, TokenType::Keyword) - end - - def tokenize(node : Ast::NumberLiteral) - add(node, TokenType::Number) - end - - def tokenize(node : Ast::Comment) - add(node, TokenType::Comment) - end - - def tokenize(node : Ast::Variable) - add(node, TokenType::Variable) - end - - def tokenize(node : Ast::TypeId) - add(node, TokenType::Type) - end - def add(from : Int32, to : Int32, type : TokenType) tokens << Token.new( type: type, @@ -94,15 +94,10 @@ module Mint to: to) end - def add(node : Ast::Node | Nil, type : TokenType) - add(node.from, node.to, type) if node - end - - def tokenize(nodes : Array(Ast::Node)) - nodes.each { |node| tokenize(node) } - end - - def tokenize(node : Ast::Node?) + def add(node : Ast::Node, type : TokenType) + return if cache.includes?(node) + add(node.from, node.to, type) + cache.add(node) end end end From ca01ad6659eab0f563d35db97c9d7be93bec3f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 13 Jun 2023 16:55:36 +0200 Subject: [PATCH 05/19] Save keywords automatically instead of manually. --- src/parser.cr | 8 ++++---- src/parsers/array_literal.cr | 2 +- src/parsers/case.cr | 4 ++-- src/parsers/component.cr | 4 ++-- src/parsers/connect.cr | 4 ++-- src/parsers/connect_variable.cr | 2 +- src/parsers/constant.cr | 2 +- src/parsers/css_font_face.cr | 2 +- src/parsers/css_keyframes.cr | 2 +- src/parsers/decode.cr | 6 +++--- src/parsers/directives/asset.cr | 2 +- src/parsers/directives/documentation.cr | 2 +- src/parsers/directives/format.cr | 2 +- src/parsers/directives/inline.cr | 2 +- src/parsers/directives/svg.cr | 2 +- src/parsers/encode.cr | 2 +- src/parsers/enum.cr | 2 +- src/parsers/for.cr | 4 ++-- src/parsers/for_condition.cr | 2 +- src/parsers/function.cr | 2 +- src/parsers/get.cr | 2 +- src/parsers/html_component.cr | 2 +- src/parsers/html_element.cr | 2 +- src/parsers/if.cr | 4 ++-- src/parsers/js.cr | 2 +- src/parsers/module.cr | 2 +- src/parsers/next_call.cr | 2 +- src/parsers/property.cr | 2 +- src/parsers/provider.cr | 2 +- src/parsers/record_definition.cr | 2 +- src/parsers/record_definition_field.cr | 2 +- src/parsers/return.cr | 2 +- src/parsers/routes.cr | 2 +- src/parsers/state.cr | 2 +- src/parsers/statement.cr | 4 ++-- src/parsers/store.cr | 2 +- src/parsers/style.cr | 2 +- src/parsers/suite.cr | 2 +- src/parsers/test.cr | 2 +- src/parsers/use.cr | 4 ++-- src/parsers/void.cr | 2 +- 41 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/parser.cr b/src/parser.cr index adbc3237d..a3d36eb73 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -173,8 +173,8 @@ module Mint # Consuming keywords # ---------------------------------------------------------------------------- - def keyword!(word, error, save : Bool = false) : Bool - keyword(word, save) || raise error + def keyword!(word, error) : Bool + keyword(word) || raise error end def keyword_ahead?(word) : Bool @@ -184,9 +184,9 @@ module Mint true end - def keyword(word, save : Bool = false) : Bool + def keyword(word) : Bool if keyword_ahead?(word) - if save + if word =~ /^[a-z]+$/ @ast.keywords << {position, position + word.size} end diff --git a/src/parsers/array_literal.cr b/src/parsers/array_literal.cr index 3d255b156..384c32ace 100644 --- a/src/parsers/array_literal.cr +++ b/src/parsers/array_literal.cr @@ -18,7 +18,7 @@ module Mint type = start do whitespace - next unless keyword("of", true) + next unless keyword "of" whitespace type_or_type_variable! ArrayLiteralExpectedTypeOrVariable end diff --git a/src/parsers/case.cr b/src/parsers/case.cr index 3c404b0db..2914179e3 100644 --- a/src/parsers/case.cr +++ b/src/parsers/case.cr @@ -8,14 +8,14 @@ module Mint def case_expression(for_css : Bool = false) : Ast::Case? start do |start_position| - next unless keyword("case", true) + next unless keyword "case" whitespace parens = char! '(' whitespace - await = keyword("await", true) + await = keyword "await" whitespace condition = expression! CaseExpectedCondition diff --git a/src/parsers/component.cr b/src/parsers/component.cr index 37a3667ec..53b5bc289 100644 --- a/src/parsers/component.cr +++ b/src/parsers/component.cr @@ -9,10 +9,10 @@ module Mint start do |start_position| comment = self.comment - global = keyword("global", true) + global = keyword "global" whitespace - next unless keyword("component", true) + next unless keyword "component" whitespace name = type_id! ComponentExpectedName diff --git a/src/parsers/connect.cr b/src/parsers/connect.cr index cc0cdf312..5229a4062 100644 --- a/src/parsers/connect.cr +++ b/src/parsers/connect.cr @@ -8,13 +8,13 @@ module Mint def connect : Ast::Connect? start do |start_position| - next unless keyword("connect", true) + next unless keyword "connect" whitespace store = type_id! ConnectExpectedType whitespace - keyword!("exposing", ConnectExpectedExposing, true) + keyword! "exposing", ConnectExpectedExposing keys = block( opening_bracket: ConnectExpectedOpeningBracket, diff --git a/src/parsers/connect_variable.cr b/src/parsers/connect_variable.cr index cd5b10cc3..62f195174 100644 --- a/src/parsers/connect_variable.cr +++ b/src/parsers/connect_variable.cr @@ -10,7 +10,7 @@ module Mint whitespace - if keyword("as", true) + if keyword "as" whitespace name = variable! ConnectVariableExpectedAs end diff --git a/src/parsers/constant.cr b/src/parsers/constant.cr index af5396c91..afe96bd84 100644 --- a/src/parsers/constant.cr +++ b/src/parsers/constant.cr @@ -9,7 +9,7 @@ module Mint comment = self.comment whitespace - next unless keyword("const", true) + next unless keyword "const" whitespace name = variable_constant! diff --git a/src/parsers/css_font_face.cr b/src/parsers/css_font_face.cr index 48775888a..ecbd9493f 100644 --- a/src/parsers/css_font_face.cr +++ b/src/parsers/css_font_face.cr @@ -5,7 +5,7 @@ module Mint def css_font_face : Ast::CssFontFace? start do |start_position| - next unless keyword("@font-face", true) + next unless keyword "@font-face" definitions = block( opening_bracket: CssFontFaceExpectedOpeningBracket, diff --git a/src/parsers/css_keyframes.cr b/src/parsers/css_keyframes.cr index d3a725d59..ddb157c5c 100644 --- a/src/parsers/css_keyframes.cr +++ b/src/parsers/css_keyframes.cr @@ -6,7 +6,7 @@ module Mint def css_keyframes : Ast::CssKeyframes? start do |start_position| - next unless keyword("@keyframes", true) + next unless keyword "@keyframes" whitespace diff --git a/src/parsers/decode.cr b/src/parsers/decode.cr index 564f27f35..473491522 100644 --- a/src/parsers/decode.cr +++ b/src/parsers/decode.cr @@ -6,15 +6,15 @@ module Mint def decode : Ast::Decode? start do |start_position| - next unless keyword("decode", true) + next unless keyword "decode" next unless whitespace? whitespace - unless keyword("as", true) + unless keyword "as" expression = expression! DecodeExpectedExpression whitespace - keyword!("as", DecodeExpectedAs, true) + keyword! "as", DecodeExpectedAs end whitespace diff --git a/src/parsers/directives/asset.cr b/src/parsers/directives/asset.cr index ed6652695..a08f51e00 100644 --- a/src/parsers/directives/asset.cr +++ b/src/parsers/directives/asset.cr @@ -6,7 +6,7 @@ module Mint def asset_directive : Ast::Directives::Asset? start do |start_position| - next unless keyword("@asset", true) + next unless keyword "@asset" char '(', AssetDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/documentation.cr b/src/parsers/directives/documentation.cr index abcc8118f..3d36434d3 100644 --- a/src/parsers/directives/documentation.cr +++ b/src/parsers/directives/documentation.cr @@ -6,7 +6,7 @@ module Mint def documentation_directive : Ast::Directives::Documentation? start do |start_position| - next unless keyword("@documentation", true) + next unless keyword "@documentation" char '(', DocumentationDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/format.cr b/src/parsers/directives/format.cr index 7cf7856f0..f16a07470 100644 --- a/src/parsers/directives/format.cr +++ b/src/parsers/directives/format.cr @@ -6,7 +6,7 @@ module Mint def format_directive : Ast::Directives::Format? start do |start_position| - next unless keyword("@format", true) + next unless keyword "@format" content = code_block( diff --git a/src/parsers/directives/inline.cr b/src/parsers/directives/inline.cr index ca84342fd..600bc4abc 100644 --- a/src/parsers/directives/inline.cr +++ b/src/parsers/directives/inline.cr @@ -6,7 +6,7 @@ module Mint def inline_directive : Ast::Directives::Inline? start do |start_position| - next unless keyword("@inline", true) + next unless keyword "@inline" char '(', InlineDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/directives/svg.cr b/src/parsers/directives/svg.cr index ab62cbde3..75b9b4acf 100644 --- a/src/parsers/directives/svg.cr +++ b/src/parsers/directives/svg.cr @@ -6,7 +6,7 @@ module Mint def svg_directive : Ast::Directives::Svg? start do |start_position| - next unless keyword("@svg", true) + next unless keyword "@svg" char '(', SvgDirectiveExpectedOpeningParentheses whitespace diff --git a/src/parsers/encode.cr b/src/parsers/encode.cr index 634ea89d0..665d8fbd1 100644 --- a/src/parsers/encode.cr +++ b/src/parsers/encode.cr @@ -4,7 +4,7 @@ module Mint def encode : Ast::Encode? start do |start_position| - next unless keyword("encode", true) + next unless keyword "encode" next unless whitespace? whitespace diff --git a/src/parsers/enum.cr b/src/parsers/enum.cr index 50a03279e..1a54106d3 100644 --- a/src/parsers/enum.cr +++ b/src/parsers/enum.cr @@ -9,7 +9,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword("enum", true) + next unless keyword "enum" whitespace name = type_id! EnumExpectedName diff --git a/src/parsers/for.cr b/src/parsers/for.cr index 54886f668..34ba64a49 100644 --- a/src/parsers/for.cr +++ b/src/parsers/for.cr @@ -9,7 +9,7 @@ module Mint def for_expression : Ast::For? start do |start_position| - next unless keyword("for", true) + next unless keyword "for" next unless whitespace? whitespace @@ -22,7 +22,7 @@ module Mint ) { variable } whitespace - keyword!("of", ForExpectedOf, true) + keyword! "of", ForExpectedOf whitespace subject = expression! ForExpectedSubject diff --git a/src/parsers/for_condition.cr b/src/parsers/for_condition.cr index cae526100..2b48d7db8 100644 --- a/src/parsers/for_condition.cr +++ b/src/parsers/for_condition.cr @@ -6,7 +6,7 @@ module Mint def for_condition : Ast::ForCondition? start do |start_position| - next unless keyword("when", true) + next unless keyword "when" whitespace condition = diff --git a/src/parsers/function.cr b/src/parsers/function.cr index 519aedcc3..d1ab46238 100644 --- a/src/parsers/function.cr +++ b/src/parsers/function.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword("fun", true) + next unless keyword "fun" whitespace name = variable! FunctionExpectedName, track: false diff --git a/src/parsers/get.cr b/src/parsers/get.cr index a61b9f29a..54593ab7b 100644 --- a/src/parsers/get.cr +++ b/src/parsers/get.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword("get", true) + next unless keyword "get" whitespace name = variable! GetExpectedName, track: false diff --git a/src/parsers/html_component.cr b/src/parsers/html_component.cr index b04851bee..4147ac54e 100644 --- a/src/parsers/html_component.cr +++ b/src/parsers/html_component.cr @@ -16,7 +16,7 @@ module Mint ref = start do whitespace - next unless keyword("as", true) + next unless keyword "as" whitespace variable! HtmlComponentExpectedReference end diff --git a/src/parsers/html_element.cr b/src/parsers/html_element.cr index 978932750..5c2db7e21 100644 --- a/src/parsers/html_element.cr +++ b/src/parsers/html_element.cr @@ -25,7 +25,7 @@ module Mint ref = start do whitespace - next unless keyword("as", true) + next unless keyword "as" whitespace variable! HtmlElementExpectedReference end diff --git a/src/parsers/if.cr b/src/parsers/if.cr index 5659683d5..54f47a607 100644 --- a/src/parsers/if.cr +++ b/src/parsers/if.cr @@ -11,7 +11,7 @@ module Mint def if_expression(for_css = false) : Ast::If? start do |start_position| - next unless keyword("if", true) + next unless keyword "if" whitespace parens = char! '(' @@ -40,7 +40,7 @@ module Mint falsy = nil whitespace - if keyword("else", true) + if keyword "else" whitespace unless falsy = if_expression(for_css: for_css) diff --git a/src/parsers/js.cr b/src/parsers/js.cr index b7edee0ac..90e5c2d32 100644 --- a/src/parsers/js.cr +++ b/src/parsers/js.cr @@ -15,7 +15,7 @@ module Mint type = start do whitespace - next unless keyword("as", true) + next unless keyword "as" whitespace type_or_type_variable! JsExpectedTypeOrVariable end diff --git a/src/parsers/module.cr b/src/parsers/module.cr index 2f82dc448..f49d8b4dc 100644 --- a/src/parsers/module.cr +++ b/src/parsers/module.cr @@ -9,7 +9,7 @@ module Mint comment = self.comment whitespace - next unless keyword("module", true) + next unless keyword "module" whitespace name = type_id! ModuleExpectedName diff --git a/src/parsers/next_call.cr b/src/parsers/next_call.cr index c42dd3595..838995c7e 100644 --- a/src/parsers/next_call.cr +++ b/src/parsers/next_call.cr @@ -4,7 +4,7 @@ module Mint def next_call : Ast::NextCall? start do |start_position| - next unless keyword("next", true) + next unless keyword "next" next unless whitespace? whitespace diff --git a/src/parsers/property.cr b/src/parsers/property.cr index 6cdf33d03..5e61b036f 100644 --- a/src/parsers/property.cr +++ b/src/parsers/property.cr @@ -11,7 +11,7 @@ module Mint start_position = position - next unless keyword("property", true) + next unless keyword "property" whitespace name = variable! PropertyExpectedName, track: false diff --git a/src/parsers/provider.cr b/src/parsers/provider.cr index f7e78b593..74709263e 100644 --- a/src/parsers/provider.cr +++ b/src/parsers/provider.cr @@ -11,7 +11,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword("provider", true) + next unless keyword "provider" whitespace name = type_id! ProviderExpectedName diff --git a/src/parsers/record_definition.cr b/src/parsers/record_definition.cr index 309306658..9da509ae4 100644 --- a/src/parsers/record_definition.cr +++ b/src/parsers/record_definition.cr @@ -8,7 +8,7 @@ module Mint start do |start_position| comment = self.comment - next unless keyword("record", true) + next unless keyword "record" whitespace name = type_id! RecordDefinitionExpectedName diff --git a/src/parsers/record_definition_field.cr b/src/parsers/record_definition_field.cr index 0f77eca02..e145bbf47 100644 --- a/src/parsers/record_definition_field.cr +++ b/src/parsers/record_definition_field.cr @@ -19,7 +19,7 @@ module Mint mapping = start do whitespace - next unless keyword("using", true) + next unless keyword "using" whitespace string_literal! RecordDefinitionFieldExpectedMapping, with_interpolation: false diff --git a/src/parsers/return.cr b/src/parsers/return.cr index ac40debfc..342e967e7 100644 --- a/src/parsers/return.cr +++ b/src/parsers/return.cr @@ -4,7 +4,7 @@ module Mint def return_call : Ast::ReturnCall? start do |start_position| - next unless keyword("return", true) + next unless keyword "return" next unless whitespace? whitespace diff --git a/src/parsers/routes.cr b/src/parsers/routes.cr index c6cbdb4e3..1392b44a2 100644 --- a/src/parsers/routes.cr +++ b/src/parsers/routes.cr @@ -6,7 +6,7 @@ module Mint def routes : Ast::Routes? start do |start_position| - next unless keyword("routes", true) + next unless keyword "routes" body = block( opening_bracket: RoutesExpectedOpeningBracket, diff --git a/src/parsers/state.cr b/src/parsers/state.cr index 12dfe60a8..2518a84c9 100644 --- a/src/parsers/state.cr +++ b/src/parsers/state.cr @@ -10,7 +10,7 @@ module Mint comment = self.comment whitespace - next unless keyword("state", true) + next unless keyword "state" whitespace name = variable! StateExpectedName diff --git a/src/parsers/statement.cr b/src/parsers/statement.cr index bac128e5e..2cdc4d09a 100644 --- a/src/parsers/statement.cr +++ b/src/parsers/statement.cr @@ -3,7 +3,7 @@ module Mint def statement : Ast::Statement? start do |start_position| target = start do - next unless keyword("let", true) + next unless keyword "let" whitespace value = variable(track: false) || @@ -19,7 +19,7 @@ module Mint end whitespace - await = keyword("await", true) + await = keyword "await" whitespace body = expression diff --git a/src/parsers/store.cr b/src/parsers/store.cr index 0963940ae..0f2f746e2 100644 --- a/src/parsers/store.cr +++ b/src/parsers/store.cr @@ -10,7 +10,7 @@ module Mint comment = self.comment whitespace - next unless keyword("store", true) + next unless keyword "store" whitespace name = type_id! StoreExpectedName diff --git a/src/parsers/style.cr b/src/parsers/style.cr index 33b2bb13f..50352f5d3 100644 --- a/src/parsers/style.cr +++ b/src/parsers/style.cr @@ -7,7 +7,7 @@ module Mint def style : Ast::Style? start do |start_position| - next unless keyword("style", true) + next unless keyword "style" whitespace name = variable_with_dashes! StyleExpectedName diff --git a/src/parsers/suite.cr b/src/parsers/suite.cr index dac4514d1..c22336995 100644 --- a/src/parsers/suite.cr +++ b/src/parsers/suite.cr @@ -7,7 +7,7 @@ module Mint def suite : Ast::Suite? start do |start_position| - next unless keyword("suite", true) + next unless keyword "suite" whitespace diff --git a/src/parsers/test.cr b/src/parsers/test.cr index 4f0b9d9c7..8765f569e 100644 --- a/src/parsers/test.cr +++ b/src/parsers/test.cr @@ -7,7 +7,7 @@ module Mint def test : Ast::Test? start do |start_position| - next unless keyword("test", true) + next unless keyword "test" whitespace diff --git a/src/parsers/use.cr b/src/parsers/use.cr index 4454a978d..1dad13e69 100644 --- a/src/parsers/use.cr +++ b/src/parsers/use.cr @@ -8,7 +8,7 @@ module Mint def use : Ast::Use? start do |start_position| - next unless keyword("use", true) + next unless keyword "use" whitespace provider = type_id! UseExpectedProvider @@ -17,7 +17,7 @@ module Mint raise UseExpectedRecord unless item = record whitespace - if keyword("when", true) + if keyword "when" condition = block( opening_bracket: UseExpectedOpeningBracket, closing_bracket: UseExpectedClosingBracket diff --git a/src/parsers/void.cr b/src/parsers/void.cr index f44a51acf..f35238456 100644 --- a/src/parsers/void.cr +++ b/src/parsers/void.cr @@ -2,7 +2,7 @@ module Mint class Parser def void : Ast::Void? start do |start_position| - next unless keyword("void", true) + next unless keyword "void" self << Ast::Void.new( from: start_position, From 6926d5768cbc8286ec879262c73d13236d972581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 13 Jun 2023 17:12:33 +0200 Subject: [PATCH 06/19] Use an array derived from the actual token types. --- src/ls/initialize.cr | 2 +- src/ls/semantic_tokens.cr | 2 +- src/semantic_tokenizer.cr | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ls/initialize.cr b/src/ls/initialize.cr index 4064d11b1..659781d9b 100644 --- a/src/ls/initialize.cr +++ b/src/ls/initialize.cr @@ -47,8 +47,8 @@ module Mint range: false, full: true, legend: LSP::SemanticTokensLegend.new( + token_types: SemanticTokenizer::TOKEN_TYPES, token_modifiers: [] of String, - token_types: ["class", "keyword", "comment", "type", "property", "number", "namespace", "variable", "string"] of String, )) capabilities = diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr index ef299f27a..bb0bdf0fb 100644 --- a/src/ls/semantic_tokens.cr +++ b/src/ls/semantic_tokens.cr @@ -25,7 +25,7 @@ module Mint type = token.type.to_s.underscore - if index = ["class", "keyword", "comment", "type", "property", "number", "namespace", "variable", "string"].index(type) + if index = SemanticTokenizer::TOKEN_TYPES.index(type) [ location.start[0] - 1, location.start[1], diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index aadd96e05..461c7b945 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -17,6 +17,8 @@ module Mint Regexp end + TOKEN_TYPES = TokenType.names.map { |name| name[0].downcase + name[1..] } + # This represents which token types are used for which node. TOKEN_MAP = { Ast::TypeVariable => TokenType::TypeParameter, From 29fb79bb4bf1c46a667cefb6746b8478eb11ed6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 08:48:04 +0200 Subject: [PATCH 07/19] Implement suggestions from code review. --- spec/language_server/semantic_tokens/record | 18 ++-- src/commands/highlight.cr | 99 ++++++++++----------- src/parser.cr | 2 +- src/parsers/type_id.cr | 4 +- src/semantic_tokenizer.cr | 1 - 5 files changed, 61 insertions(+), 63 deletions(-) diff --git a/spec/language_server/semantic_tokens/record b/spec/language_server/semantic_tokens/record index a2b194b01..deadce430 100644 --- a/spec/language_server/semantic_tokens/record +++ b/spec/language_server/semantic_tokens/record @@ -38,47 +38,47 @@ record Article { 0, 0, 20, - 2, + 5, 0, 1, 0, 6, - 1, + 4, 0, 0, 7, 7, - 3, + 1, 0, 1, 2, 2, - 7, + 6, 0, 0, 5, 6, - 3, + 1, 0, 1, 2, 11, - 7, + 6, 0, 0, 14, 6, - 3, + 1, 0, 1, 2, 5, - 7, + 6, 0, 0, 8, 6, - 3, + 1, 0 ] }, diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr index 3950de5df..06bba8ec6 100644 --- a/src/commands/highlight.cr +++ b/src/commands/highlight.cr @@ -9,65 +9,64 @@ module Mint description: "The path to the file" def run - if path = arguments.path - ast = - Parser.parse(path) + return unless path = arguments.path - tokenizer = SemanticTokenizer.new - tokenizer.tokenize(ast) + ast = + Parser.parse(path) - parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) - contents = File.read(path) - position = 0 + tokenizer = SemanticTokenizer.new + tokenizer.tokenize(ast) - tokenizer.tokens.sort_by(&.from).each do |token| - if token.from > position - parts << contents[position, token.from - position] - end + parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) + contents = File.read(path) + position = 0 - parts << {contents[token.from, token.to - token.from], token.type} - position = token.to + tokenizer.tokens.sort_by(&.from).each do |token| + if token.from > position + parts << contents[position, token.from - position] end - if position < contents.size - parts << contents[position, contents.size] - end + parts << {contents[token.from, token.to - token.from], token.type} + position = token.to + end - result = parts.reduce("") do |memo, item| - memo + case item - in String - item - in Tuple(String, SemanticTokenizer::TokenType) - case item[1] - in SemanticTokenizer::TokenType::Type - item[0].colorize(:yellow) - in SemanticTokenizer::TokenType::TypeParameter - item[0].colorize(:light_yellow) - in SemanticTokenizer::TokenType::Variable - item[0].colorize(:dark_gray) - in SemanticTokenizer::TokenType::Namespace - item[0].colorize(:light_blue) - in SemanticTokenizer::TokenType::Keyword - item[0].colorize(:magenta) - in SemanticTokenizer::TokenType::Property - item[0].colorize(:dark_gray).mode(:underline) - in SemanticTokenizer::TokenType::Comment - item[0].colorize(:light_gray) - in SemanticTokenizer::TokenType::String - item[0].colorize(:green) - in SemanticTokenizer::TokenType::Number - item[0].colorize(:red) - in SemanticTokenizer::TokenType::Regexp - item[0].colorize.fore(:white).back(:red) - in SemanticTokenizer::TokenType::Operator - item[0].colorize.fore(:white).back(:red) - end.to_s - # %(#{item[0]}) - end - end + if position < contents.size + parts << contents[position, contents.size] + end - print result + result = parts.reduce("") do |memo, item| + memo + case item + in String + item + in Tuple(String, SemanticTokenizer::TokenType) + case item[1] + in .type? + item[0].colorize(:yellow) + in .type_parameter? + item[0].colorize(:light_yellow) + in .variable? + item[0].colorize(:dark_gray) + in .namespace? + item[0].colorize(:light_blue) + in .keyword? + item[0].colorize(:magenta) + in .property? + item[0].colorize(:dark_gray).mode(:underline) + in .comment? + item[0].colorize(:light_gray) + in .string? + item[0].colorize(:green) + in .number? + item[0].colorize(:red) + in .regexp? + item[0].colorize.fore(:white).back(:red) + in .operator? + item[0].colorize(:light_magenta) + end.to_s + end end + + print result end end end diff --git a/src/parser.cr b/src/parser.cr index a3d36eb73..b72611838 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -186,7 +186,7 @@ module Mint def keyword(word) : Bool if keyword_ahead?(word) - if word =~ /^[a-z]+$/ + if word.chars.all?(&.ascii_lowercase?) && !word.blank? && word != "or" @ast.keywords << {position, position + word.size} end diff --git a/src/parsers/type_id.cr b/src/parsers/type_id.cr index aefea45d7..3e251c2b0 100644 --- a/src/parsers/type_id.cr +++ b/src/parsers/type_id.cr @@ -10,7 +10,7 @@ module Mint raise error unless value if char! '.' - other = type_id!(error, false) + other = type_id!(error, track: false) value += ".#{other.value}" end @@ -38,7 +38,7 @@ module Mint if char == '.' other = start do step - next_part = type_id(false) + next_part = type_id(track: false) next unless next_part next_part end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index 461c7b945..869b78972 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -23,7 +23,6 @@ module Mint TOKEN_MAP = { Ast::TypeVariable => TokenType::TypeParameter, Ast::Variable => TokenType::Variable, - Ast::BoolLiteral => TokenType::Keyword, Ast::Comment => TokenType::Comment, Ast::StringLiteral => TokenType::String, Ast::NumberLiteral => TokenType::Number, From 6b25fc8ff7902bc35dd923a96b86eaa1524688a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 12:25:21 +0200 Subject: [PATCH 08/19] Update src/ls/semantic_tokens.cr Co-authored-by: jansul --- src/ls/semantic_tokens.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr index bb0bdf0fb..8824ee851 100644 --- a/src/ls/semantic_tokens.cr +++ b/src/ls/semantic_tokens.cr @@ -1,6 +1,6 @@ module Mint module LS - # This is the class that handles the "textDocument/hover" request. + # This is the class that handles the "textDocument/semanticTokens/full" request. class SemanticTokens < LSP::RequestMessage property params : LSP::SemanticTokensParams From 0eea575a31a2740bbedfb7d8e807764ca95c9d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 12:58:11 +0200 Subject: [PATCH 09/19] Implement HTML highlighting. --- src/commands/highlight.cr | 67 ++++----------------------------------- src/semantic_tokenizer.cr | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr index 06bba8ec6..5825d7a11 100644 --- a/src/commands/highlight.cr +++ b/src/commands/highlight.cr @@ -3,70 +3,17 @@ module Mint class Highlight < Admiral::Command include Command - define_help description: "Returns the syntax highlighted version of the given file as HTML" + define_help description: "Returns the syntax highlighted version of the given file" - define_argument path, - description: "The path to the file" + define_argument path, description: "The path to the file" + + define_flag html : Bool, + description: "If specified, print the highlighted code as HTML", + default: false def run return unless path = arguments.path - - ast = - Parser.parse(path) - - tokenizer = SemanticTokenizer.new - tokenizer.tokenize(ast) - - parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) - contents = File.read(path) - position = 0 - - tokenizer.tokens.sort_by(&.from).each do |token| - if token.from > position - parts << contents[position, token.from - position] - end - - parts << {contents[token.from, token.to - token.from], token.type} - position = token.to - end - - if position < contents.size - parts << contents[position, contents.size] - end - - result = parts.reduce("") do |memo, item| - memo + case item - in String - item - in Tuple(String, SemanticTokenizer::TokenType) - case item[1] - in .type? - item[0].colorize(:yellow) - in .type_parameter? - item[0].colorize(:light_yellow) - in .variable? - item[0].colorize(:dark_gray) - in .namespace? - item[0].colorize(:light_blue) - in .keyword? - item[0].colorize(:magenta) - in .property? - item[0].colorize(:dark_gray).mode(:underline) - in .comment? - item[0].colorize(:light_gray) - in .string? - item[0].colorize(:green) - in .number? - item[0].colorize(:red) - in .regexp? - item[0].colorize.fore(:white).back(:red) - in .operator? - item[0].colorize(:light_magenta) - end.to_s - end - end - - print result + print SemanticTokenizer.highlight(path, flags.html) end end end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index 869b78972..57104645f 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -25,6 +25,7 @@ module Mint Ast::Variable => TokenType::Variable, Ast::Comment => TokenType::Comment, Ast::StringLiteral => TokenType::String, + Ast::RegexpLiteral => TokenType::Regexp, Ast::NumberLiteral => TokenType::Number, Ast::TypeId => TokenType::Type, } @@ -42,6 +43,71 @@ module Mint # This is where the resulting tokens are stored. getter tokens : Array(Token) = [] of Token + def self.highlight(path : String, html : Bool = false) + ast = + Parser.parse(path) + + tokenizer = self.new + tokenizer.tokenize(ast) + + parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) + contents = ast.nodes.first.input.input + position = 0 + + tokenizer.tokens.sort_by(&.from).each do |token| + if token.from > position + parts << contents[position, token.from - position] + end + + parts << {contents[token.from, token.to - token.from], token.type} + position = token.to + end + + if position < contents.size + parts << contents[position, contents.size] + end + + parts.reduce("") do |memo, item| + memo + case item + in String + if html + HTML.escape(item) + else + item + end + in Tuple(String, SemanticTokenizer::TokenType) + if html + "#{HTML.escape(item[0])}" + else + case item[1] + in .type? + item[0].colorize(:yellow) + in .type_parameter? + item[0].colorize(:light_yellow) + in .variable? + item[0].colorize(:dark_gray) + in .namespace? + item[0].colorize(:light_blue) + in .keyword? + item[0].colorize(:magenta) + in .property? + item[0].colorize(:dark_gray).mode(:underline) + in .comment? + item[0].colorize(:light_gray) + in .string? + item[0].colorize(:green) + in .number? + item[0].colorize(:red) + in .regexp? + item[0].colorize(:light_red) + in .operator? + item[0].colorize(:light_magenta) + end.to_s + end + end + end + end + def tokenize(ast : Ast) # We add the operators and keywords directly from the AST ast.operators.each { |(from, to)| add(from, to, TokenType::Operator) } From 1cd96da0d2bc29cb1fdca5fba3004372bd19e636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 14:28:41 +0200 Subject: [PATCH 10/19] Implement highlight directive. --- src/ast/directives/highlight.cr | 15 ++++++++++ src/compilers/directives/highlight.cr | 36 +++++++++++++++++++++++ src/formatters/directives/highlight.cr | 7 +++++ src/ls/semantic_tokens.cr | 1 + src/parsers/basic_expression.cr | 1 + src/parsers/code_block.cr | 13 ++++++++ src/parsers/directives/highlight.cr | 25 ++++++++++++++++ src/semantic_tokenizer.cr | 15 +++++++--- src/type_checkers/directives/highlight.cr | 7 +++++ 9 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 src/ast/directives/highlight.cr create mode 100644 src/compilers/directives/highlight.cr create mode 100644 src/formatters/directives/highlight.cr create mode 100644 src/parsers/directives/highlight.cr create mode 100644 src/type_checkers/directives/highlight.cr diff --git a/src/ast/directives/highlight.cr b/src/ast/directives/highlight.cr new file mode 100644 index 000000000..3f4ac2ba0 --- /dev/null +++ b/src/ast/directives/highlight.cr @@ -0,0 +1,15 @@ +module Mint + class Ast + module Directives + class Highlight < Node + getter content + + def initialize(@content : Block, + @input : Data, + @from : Int32, + @to : Int32) + end + end + end + end +end diff --git a/src/compilers/directives/highlight.cr b/src/compilers/directives/highlight.cr new file mode 100644 index 000000000..50761c3af --- /dev/null +++ b/src/compilers/directives/highlight.cr @@ -0,0 +1,36 @@ +module Mint + class Compiler + def _compile(node : Ast::Directives::Highlight) : String + content = + compile node.content + + formatted = + Formatter.new.format(node.content, Formatter::BlockFormat::Naked) + + parser = Parser.new(formatted, "source.mint") + parser.code_block_naked + + contents, parts = + SemanticTokenizer.tokenize(parser.ast) + + mapped = + parts.map do |item| + case item + in String + "`#{skip { escape_for_javascript(item) }}`" + in Tuple(String, SemanticTokenizer::TokenType) + "_h('span', { className: '#{item[1].to_s.underscore}' }, [`#{skip { escape_for_javascript(item[0]) }}`])" + end + end + + "[#{content}, _h(React.Fragment, {}, [#{mapped.join(",\n")}])]" + end + + def escape_for_javascript(value : String) + value + .gsub('\\', "\\\\") + .gsub('`', "\\`") + .gsub("${", "\\${") + end + end +end diff --git a/src/formatters/directives/highlight.cr b/src/formatters/directives/highlight.cr new file mode 100644 index 000000000..ba0408468 --- /dev/null +++ b/src/formatters/directives/highlight.cr @@ -0,0 +1,7 @@ +module Mint + class Formatter + def format(node : Ast::Directives::Highlight) + "@highlight #{format(node.content)}" + end + end +end diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr index 8824ee851..59cd7f09f 100644 --- a/src/ls/semantic_tokens.cr +++ b/src/ls/semantic_tokens.cr @@ -14,6 +14,7 @@ module Mint # This is used later on to convert the line/column of each token input = ast.nodes.first.input + tokenizer = SemanticTokenizer.new tokenizer.tokenize(ast) diff --git a/src/parsers/basic_expression.cr b/src/parsers/basic_expression.cr index 49a3e701b..051489ac5 100644 --- a/src/parsers/basic_expression.cr +++ b/src/parsers/basic_expression.cr @@ -3,6 +3,7 @@ module Mint # NOTE: The order of the parsing is important! def basic_expression : Ast::Expression? format_directive || + highlight_directive || documentation_directive || svg_directive || asset_directive || diff --git a/src/parsers/code_block.cr b/src/parsers/code_block.cr index 6c3717e27..1b943e522 100644 --- a/src/parsers/code_block.cr +++ b/src/parsers/code_block.cr @@ -1,5 +1,18 @@ module Mint class Parser + def code_block_naked : Ast::Block? + start do |start_position| + statements = + many { comment || statement } + + self << Ast::Block.new( + statements: statements, + from: start_position, + to: position, + input: data) if statements + end + end + def code_block : Ast::Block? start do |start_position| statements = diff --git a/src/parsers/directives/highlight.cr b/src/parsers/directives/highlight.cr new file mode 100644 index 000000000..e273354a7 --- /dev/null +++ b/src/parsers/directives/highlight.cr @@ -0,0 +1,25 @@ +module Mint + class Parser + syntax_error FormatDirectiveExpectedOpeningBracket + syntax_error FormatDirectiveExpectedClosingBracket + syntax_error FormatDirectiveExpectedExpression + + def highlight_directive : Ast::Directives::Highlight? + start do |start_position| + next unless keyword "@highlight" + + content = + code_block( + opening_bracket: FormatDirectiveExpectedOpeningBracket, + closing_bracket: FormatDirectiveExpectedClosingBracket, + statement_error: FormatDirectiveExpectedExpression) + + self << Ast::Directives::Highlight.new( + from: start_position, + content: content, + to: position, + input: data) + end + end + end +end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index 57104645f..957550a0e 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -43,10 +43,7 @@ module Mint # This is where the resulting tokens are stored. getter tokens : Array(Token) = [] of Token - def self.highlight(path : String, html : Bool = false) - ast = - Parser.parse(path) - + def self.tokenize(ast : Ast) tokenizer = self.new tokenizer.tokenize(ast) @@ -67,6 +64,16 @@ module Mint parts << contents[position, contents.size] end + {contents, parts} + end + + def self.highlight(path : String, html : Bool = false) + ast = + Parser.parse(path) + + contents, parts = + tokenize(ast) + parts.reduce("") do |memo, item| memo + case item in String diff --git a/src/type_checkers/directives/highlight.cr b/src/type_checkers/directives/highlight.cr new file mode 100644 index 000000000..9e47bf932 --- /dev/null +++ b/src/type_checkers/directives/highlight.cr @@ -0,0 +1,7 @@ +module Mint + class TypeChecker + def check(node : Ast::Directives::Highlight) : Checkable + Type.new("Tuple", [resolve(node.content), HTML] of Checkable) + end + end +end From 529117dc60606248394c07663c992fd428aa2827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 15:02:12 +0200 Subject: [PATCH 11/19] Avoid unnecessary interations. --- src/compilers/directives/highlight.cr | 2 +- src/parser.cr | 9 +++------ src/semantic_tokenizer.cr | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/compilers/directives/highlight.cr b/src/compilers/directives/highlight.cr index 50761c3af..605b3e027 100644 --- a/src/compilers/directives/highlight.cr +++ b/src/compilers/directives/highlight.cr @@ -10,7 +10,7 @@ module Mint parser = Parser.new(formatted, "source.mint") parser.code_block_naked - contents, parts = + parts = SemanticTokenizer.tokenize(parser.ast) mapped = diff --git a/src/parser.cr b/src/parser.cr index b72611838..c2d9083a8 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -22,23 +22,20 @@ module Mint def start(&) start_position = position + node_size = ast.nodes.size begin node = yield position @position = start_position unless node - clear_nodes(start_position) unless node + ast.nodes.delete_at(node_size...) unless node node rescue error : Error @position = start_position - clear_nodes(start_position) + ast.nodes.delete_at(node_size...) raise error end end - def clear_nodes(from_position) - ast.nodes.reject! { |node| node.from >= from_position } - end - def step @position += 1 end diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index 957550a0e..a6eb51cb6 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -64,14 +64,14 @@ module Mint parts << contents[position, contents.size] end - {contents, parts} + parts end def self.highlight(path : String, html : Bool = false) ast = Parser.parse(path) - contents, parts = + parts = tokenize(ast) parts.reduce("") do |memo, item| From 79c3f11181a688bed6d9f2f31c601a5e7a649e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Wed, 14 Jun 2023 15:42:30 +0200 Subject: [PATCH 12/19] Implement suggestions from code review. --- src/commands/highlight.cr | 2 +- src/compilers/directives/highlight.cr | 4 +- src/ls/definition/type_id.cr | 20 +++----- src/ls/initialize.cr | 4 +- src/semantic_tokenizer.cr | 72 +++++++++++++-------------- 5 files changed, 46 insertions(+), 56 deletions(-) diff --git a/src/commands/highlight.cr b/src/commands/highlight.cr index 5825d7a11..6b1195f0a 100644 --- a/src/commands/highlight.cr +++ b/src/commands/highlight.cr @@ -13,7 +13,7 @@ module Mint def run return unless path = arguments.path - print SemanticTokenizer.highlight(path, flags.html) + puts SemanticTokenizer.highlight(path, flags.html) end end end diff --git a/src/compilers/directives/highlight.cr b/src/compilers/directives/highlight.cr index 605b3e027..8690696a9 100644 --- a/src/compilers/directives/highlight.cr +++ b/src/compilers/directives/highlight.cr @@ -18,8 +18,8 @@ module Mint case item in String "`#{skip { escape_for_javascript(item) }}`" - in Tuple(String, SemanticTokenizer::TokenType) - "_h('span', { className: '#{item[1].to_s.underscore}' }, [`#{skip { escape_for_javascript(item[0]) }}`])" + in Tuple(SemanticTokenizer::TokenType, String) + "_h('span', { className: '#{item[0].to_s.underscore}' }, [`#{skip { escape_for_javascript(item[1]) }}`])" end end diff --git a/src/ls/definition/type_id.cr b/src/ls/definition/type_id.cr index f8b15a051..eed5a2a24 100644 --- a/src/ls/definition/type_id.cr +++ b/src/ls/definition/type_id.cr @@ -9,20 +9,14 @@ module Mint find_component(workspace, node.value) if found.nil? && (next_node = stack[1]) - definition(next_node, server, workspace, stack) - else - return if Core.ast.nodes.includes?(found) + return definition(next_node, server, workspace, stack) + end + + return if Core.ast.nodes.includes?(found) - case found - when Ast::Store - location_link server, node, found.name, found - when Ast::Enum - location_link server, node, found.name, found - when Ast::Component - location_link server, node, found.name, found - when Ast::RecordDefinition - location_link server, node, found.name, found - end + case found + when Ast::Store, Ast::Enum, Ast::Component, Ast::RecordDefinition + location_link server, node, found.name, found end end end diff --git a/src/ls/initialize.cr b/src/ls/initialize.cr index 659781d9b..4a712d096 100644 --- a/src/ls/initialize.cr +++ b/src/ls/initialize.cr @@ -42,7 +42,7 @@ module Mint change_notifications: false, supported: false)) - semantic_tokens_options = + semantic_tokens_provider = LSP::SemanticTokensOptions.new( range: false, full: true, @@ -55,7 +55,7 @@ module Mint LSP::ServerCapabilities.new( document_on_type_formatting_provider: document_on_type_formatting_provider, execute_command_provider: execute_command_provider, - semantic_tokens_provider: semantic_tokens_options, + semantic_tokens_provider: semantic_tokens_provider, signature_help_provider: signature_help_provider, document_link_provider: document_link_provider, completion_provider: completion_provider, diff --git a/src/semantic_tokenizer.cr b/src/semantic_tokenizer.cr index a6eb51cb6..e5c183f7f 100644 --- a/src/semantic_tokenizer.cr +++ b/src/semantic_tokenizer.cr @@ -38,16 +38,16 @@ module Mint to : Int32 # We keep a cache of all tokenized nodes to avoid duplications - getter cache : Set(Ast::Node) = Set(Ast::Node).new + getter cache = Set(Ast::Node).new # This is where the resulting tokens are stored. - getter tokens : Array(Token) = [] of Token + getter tokens = [] of Token def self.tokenize(ast : Ast) tokenizer = self.new tokenizer.tokenize(ast) - parts = [] of String | Tuple(String, SemanticTokenizer::TokenType) + parts = [] of String | Tuple(SemanticTokenizer::TokenType, String) contents = ast.nodes.first.input.input position = 0 @@ -56,7 +56,7 @@ module Mint parts << contents[position, token.from - position] end - parts << {contents[token.from, token.to - token.from], token.type} + parts << {token.type, contents[token.from, token.to - token.from]} position = token.to end @@ -74,43 +74,39 @@ module Mint parts = tokenize(ast) - parts.reduce("") do |memo, item| - memo + case item + parts.join do |item| + case item in String + html ? HTML.escape(item) : item + in Tuple(SemanticTokenizer::TokenType, String) if html - HTML.escape(item) - else - item - end - in Tuple(String, SemanticTokenizer::TokenType) - if html - "#{HTML.escape(item[0])}" - else - case item[1] - in .type? - item[0].colorize(:yellow) - in .type_parameter? - item[0].colorize(:light_yellow) - in .variable? - item[0].colorize(:dark_gray) - in .namespace? - item[0].colorize(:light_blue) - in .keyword? - item[0].colorize(:magenta) - in .property? - item[0].colorize(:dark_gray).mode(:underline) - in .comment? - item[0].colorize(:light_gray) - in .string? - item[0].colorize(:green) - in .number? - item[0].colorize(:red) - in .regexp? - item[0].colorize(:light_red) - in .operator? - item[0].colorize(:light_magenta) - end.to_s + next "#{HTML.escape(item[1])}" end + + case item[0] + in .type? + item[1].colorize(:yellow) + in .type_parameter? + item[1].colorize(:light_yellow) + in .variable? + item[1].colorize(:dark_gray) + in .namespace? + item[1].colorize(:light_blue) + in .keyword? + item[1].colorize(:magenta) + in .property? + item[1].colorize(:dark_gray).mode(:underline) + in .comment? + item[1].colorize(:light_gray) + in .string? + item[1].colorize(:green) + in .number? + item[1].colorize(:red) + in .regexp? + item[1].colorize(:light_red) + in .operator? + item[1].colorize(:light_magenta) + end.to_s end end end From 6faec2606db1691a568a3282dadc0afc0ba9d635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Sun, 25 Jun 2023 13:45:25 +0200 Subject: [PATCH 13/19] Use the ast from the workspace semantic tokens. --- src/ls/semantic_tokens.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ls/semantic_tokens.cr b/src/ls/semantic_tokens.cr index 59cd7f09f..aeb24a56d 100644 --- a/src/ls/semantic_tokens.cr +++ b/src/ls/semantic_tokens.cr @@ -9,7 +9,7 @@ module Mint URI.parse(params.text_document.uri) ast = - Parser.parse(uri.path.to_s) + Workspace[uri.path.to_s][uri.path.to_s] # This is used later on to convert the line/column of each token input = From 37db9b699eb2e5dfb46426420677f9e37a194187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Mon, 26 Jun 2023 14:56:37 +0200 Subject: [PATCH 14/19] Implementation of localization language structures. --- core/source/Locale.mint | 19 +++++++ documentation/Language/Locale.md | 47 +++++++++++++++++ spec/compilers/locale_key | 25 +++++++++ spec/formatters/locale | 11 ++++ spec/formatters/locale_key | 23 ++++++++ spec/parsers/locale_key_spec.cr | 11 ++++ spec/parsers/locale_spec.cr | 16 ++++++ spec/type_checking/locale_key | 43 +++++++++++++++ src/ast.cr | 22 ++++++-- src/ast/component.cr | 4 +- src/ast/locale.cr | 15 ++++++ src/ast/locale_key.cr | 13 +++++ src/compiler.cr | 2 +- src/compilers/component.cr | 5 ++ src/compilers/locale_key.cr | 7 +++ src/compilers/top_level.cr | 52 +++++++++++++++++++ src/formatters/locale.cr | 10 ++++ src/formatters/locale_key.cr | 7 +++ src/formatters/top_level.cr | 3 +- src/messages/expected_end_of_file.cr | 1 + .../locale_expected_closing_bracket.cr | 7 +++ src/messages/locale_expected_language.cr | 13 +++++ .../locale_expected_opening_bracket.cr | 7 +++ src/messages/translation_mismatch.cr | 15 ++++++ src/messages/translation_missing.cr | 10 ++++ src/messages/translation_not_translated.cr | 12 +++++ src/parser.cr | 1 + src/parsers/basic_expression.cr | 1 + src/parsers/component.cr | 4 +- src/parsers/locale.cr | 39 ++++++++++++++ src/parsers/locale_key.cr | 25 +++++++++ src/parsers/top_level.cr | 3 ++ src/type_checker.cr | 6 ++- src/type_checker/artifacts.cr | 3 +- src/type_checker/scope.cr | 8 +++ src/type_checkers/locale.cr | 28 ++++++++++ src/type_checkers/locale_key.cr | 48 +++++++++++++++++ 37 files changed, 557 insertions(+), 9 deletions(-) create mode 100644 core/source/Locale.mint create mode 100644 documentation/Language/Locale.md create mode 100644 spec/compilers/locale_key create mode 100644 spec/formatters/locale create mode 100644 spec/formatters/locale_key create mode 100644 spec/parsers/locale_key_spec.cr create mode 100644 spec/parsers/locale_spec.cr create mode 100644 spec/type_checking/locale_key create mode 100644 src/ast/locale.cr create mode 100644 src/ast/locale_key.cr create mode 100644 src/compilers/locale_key.cr create mode 100644 src/formatters/locale.cr create mode 100644 src/formatters/locale_key.cr create mode 100644 src/messages/locale_expected_closing_bracket.cr create mode 100644 src/messages/locale_expected_language.cr create mode 100644 src/messages/locale_expected_opening_bracket.cr create mode 100644 src/messages/translation_mismatch.cr create mode 100644 src/messages/translation_missing.cr create mode 100644 src/messages/translation_not_translated.cr create mode 100644 src/parsers/locale.cr create mode 100644 src/parsers/locale_key.cr create mode 100644 src/type_checkers/locale.cr create mode 100644 src/type_checkers/locale_key.cr diff --git a/core/source/Locale.mint b/core/source/Locale.mint new file mode 100644 index 000000000..4942067ab --- /dev/null +++ b/core/source/Locale.mint @@ -0,0 +1,19 @@ +module Locale { + /* Sets the current locale. */ + fun set (locale : String) : Bool { + `_L.set(#{locale})` + } + + /* Returns the current locale. */ + fun get : Maybe(String) { + ` + (() => { + if (_L.locale) { + return #{Maybe::Just(`_L.locale`)} + } else { + return #{Maybe::Nothing} + } + })() + ` + } +} diff --git a/documentation/Language/Locale.md b/documentation/Language/Locale.md new file mode 100644 index 000000000..6c66d4446 --- /dev/null +++ b/documentation/Language/Locale.md @@ -0,0 +1,47 @@ +# Locale + +This feature of the language allows specifing localization tokens and values for languages indentified by [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) codes. + +```mint +locale en { + ui: { + buttons: { + ok: "OK" + } + } +} + +locale hu { + ui: { + buttons: { + ok: "Rendben" + } + } +} +``` + +A locale consists of a tree structure where the keys are lowercase identifiers and the values are expressions. + +To localize a value you need to use the locale token: + +``` +:ui.buttons.ok +``` + +or if it's a function if can be called: + +``` +:ui.buttons.ok(param1, param2) +``` + +To get and set the current locale the `Locale` module can be used: + +``` +Locale.set("en") +Locale.get() // Maybe::Just("en") +``` + +Every translation is typed checked: + +* all translations of the same key must have the same type +* locale keys must have translations in every defined language diff --git a/spec/compilers/locale_key b/spec/compilers/locale_key new file mode 100644 index 000000000..6521ef9e7 --- /dev/null +++ b/spec/compilers/locale_key @@ -0,0 +1,25 @@ +locale en { + test: "Hello" +} + +component Main { + fun render : String { + :test + } +} +-------------------------------------------------------------------------------- +class A extends _C { + componentWillUnmount() { + _L._unsubscribe(this); + } + + componentDidMount() { + _L._subscribe(this); + } + + render() { + return _L.t("test"); + } +}; + +A.displayName = "Main"; diff --git a/spec/formatters/locale b/spec/formatters/locale new file mode 100644 index 000000000..9a5c5962d --- /dev/null +++ b/spec/formatters/locale @@ -0,0 +1,11 @@ +locale en { + ui: { + button: { + ok: "OK" + } + } +} +-------------------------------------------------------------------------------- +locale en { + ui: { button: { ok: "OK" } } +} diff --git a/spec/formatters/locale_key b/spec/formatters/locale_key new file mode 100644 index 000000000..315e55b3e --- /dev/null +++ b/spec/formatters/locale_key @@ -0,0 +1,23 @@ +locale en { + ui: { + button: { + ok: "OK" + } + } +} + +component Main { + fun render : String { + :ui.button.ok + } +} +-------------------------------------------------------------------------------- +locale en { + ui: { button: { ok: "OK" } } +} + +component Main { + fun render : String { + :ui.button.ok + } +} diff --git a/spec/parsers/locale_key_spec.cr b/spec/parsers/locale_key_spec.cr new file mode 100644 index 000000000..b1c314824 --- /dev/null +++ b/spec/parsers/locale_key_spec.cr @@ -0,0 +1,11 @@ +require "../spec_helper" + +describe "Locale" do + subject locale_key + + expect_ignore "comp" + expect_ignore "asd" + expect_ignore ":" + + expect_ok ":ui.button.ok" +end diff --git a/spec/parsers/locale_spec.cr b/spec/parsers/locale_spec.cr new file mode 100644 index 000000000..4c7b0f667 --- /dev/null +++ b/spec/parsers/locale_spec.cr @@ -0,0 +1,16 @@ +require "../spec_helper" + +describe "Locale" do + subject locale + + expect_ignore "comp" + expect_ignore "asd" + + expect_error "locale", Mint::Parser::LocaleExpectedLanguage + expect_error "locale{", Mint::Parser::LocaleExpectedLanguage + expect_error "locale ", Mint::Parser::LocaleExpectedLanguage + expect_error "locale en", Mint::Parser::LocaleExpectedOpeningBracket + expect_error "locale en {", Mint::Parser::LocaleExpectedClosingBracket + + expect_ok "locale en { a: \"\" }" +end diff --git a/spec/type_checking/locale_key b/spec/type_checking/locale_key new file mode 100644 index 000000000..08b2eb7cf --- /dev/null +++ b/spec/type_checking/locale_key @@ -0,0 +1,43 @@ +locale en { + test: "" +} + +component Main { + fun render : String { + :test + } +} +-------------------------------------------------------------TranslationMissing +component Main { + fun render : String { + :test + } +} +-------------------------------------------------------TranslationNotTranslated +locale en { + test: "" +} + +locale hu { + +} + +component Main { + fun render : String { + :test + } +} +------------------------------------------------------------TranslationMismatch +locale en { + test: "" +} + +locale hu { + test: 0 +} + +component Main { + fun render : String { + :test + } +} diff --git a/src/ast.cr b/src/ast.cr index 539e32c83..9b1b591bc 100644 --- a/src/ast.cr +++ b/src/ast.cr @@ -30,18 +30,21 @@ module Mint If | Js - getter components, modules, records, stores, routes, providers - getter suites, enums, comments, nodes, unified_modules, keywords - getter operators + getter components, modules, records, stores, routes, providers, operators + getter suites, enums, comments, nodes, keywords, locales + + getter unified_modules, unified_locales def initialize(@operators = [] of Tuple(Int32, Int32), @keywords = [] of Tuple(Int32, Int32), @records = [] of RecordDefinition, @unified_modules = [] of Module, + @unified_locales = [] of Locale, @components = [] of Component, @providers = [] of Provider, @comments = [] of Comment, @modules = [] of Module, + @locales = [] of Locale, @routes = [] of Routes, @suites = [] of Suite, @stores = [] of Store, @@ -77,6 +80,7 @@ module Mint @comments.concat ast.comments @modules.concat ast.modules @records.concat ast.records + @locales.concat ast.locales @stores.concat ast.stores @routes.concat ast.routes @suites.concat ast.suites @@ -111,6 +115,18 @@ module Mint ) end + @unified_locales = + @locales + .group_by(&.language) + .map do |_, locales| + Locale.new( + input: Data.new(input: "", file: ""), + fields: locales.flat_map(&.fields), + language: locales.first.language, + comment: nil, + from: 0, + to: 0) + end self end end diff --git a/src/ast/component.cr b/src/ast/component.cr index f58381019..5351d5c45 100644 --- a/src/ast/component.cr +++ b/src/ast/component.cr @@ -3,7 +3,8 @@ module Mint class Component < Node getter properties, connects, styles, states, comments getter functions, gets, uses, name, comment, refs, constants - getter? global + + getter? global, locales def initialize(@refs : Array(Tuple(Variable, Node)), @properties : Array(Property), @@ -16,6 +17,7 @@ module Mint @comment : Comment?, @gets : Array(Get), @uses : Array(Use), + @locales : Bool, @global : Bool, @name : TypeId, @input : Data, diff --git a/src/ast/locale.cr b/src/ast/locale.cr new file mode 100644 index 000000000..027c2c210 --- /dev/null +++ b/src/ast/locale.cr @@ -0,0 +1,15 @@ +module Mint + class Ast + class Locale < Node + getter fields, comment, language + + def initialize(@fields : Array(RecordField), + @comment : Comment?, + @language : String, + @input : Data, + @from : Int32, + @to : Int32) + end + end + end +end diff --git a/src/ast/locale_key.cr b/src/ast/locale_key.cr new file mode 100644 index 000000000..742aa0bfc --- /dev/null +++ b/src/ast/locale_key.cr @@ -0,0 +1,13 @@ +module Mint + class Ast + class LocaleKey < Node + getter value + + def initialize(@value : String, + @input : Data, + @from : Int32, + @to : Int32) + end + end + end +end diff --git a/src/compiler.cr b/src/compiler.cr index ca7ebd9e4..1d08c956b 100644 --- a/src/compiler.cr +++ b/src/compiler.cr @@ -4,7 +4,7 @@ module Mint delegate lookups, checked, cache, component_records, to: @artifacts delegate ast, types, variables, resolve_order, to: @artifacts - delegate record_field_lookup, to: @artifacts + delegate record_field_lookup, locales, to: @artifacts getter js, style_builder, static_components, static_components_pool getter build, relative diff --git a/src/compilers/component.cr b/src/compilers/component.cr index 85e0a0874..78363634e 100644 --- a/src/compilers/component.cr +++ b/src/compilers/component.cr @@ -143,6 +143,11 @@ module Mint "componentDidMount" => %w[], } + if node.locales? + heads["componentWillUnmount"] << "_L._unsubscribe(this)" + heads["componentDidMount"] << "_L._subscribe(this)" + end + node.connects.each do |item| store = ast.stores.find(&.name.value.==(item.store.value)) diff --git a/src/compilers/locale_key.cr b/src/compilers/locale_key.cr new file mode 100644 index 000000000..90d655efd --- /dev/null +++ b/src/compilers/locale_key.cr @@ -0,0 +1,7 @@ +module Mint + class Compiler + def _compile(node : Ast::LocaleKey) : String + %(_L.t("#{node.value}")) + end + end +end diff --git a/src/compilers/top_level.cr b/src/compilers/top_level.cr index 4c9eb1cb0..c4cfe6416 100644 --- a/src/compilers/top_level.cr +++ b/src/compilers/top_level.cr @@ -208,6 +208,22 @@ module Mint end end + def compiled_locales + mapped = + locales.each_with_object({} of String => Hash(String, String)) do |(key, data), memo| + data.each do |language, node| + if node.in?(checked) + memo[language] ||= {} of String => String + memo[language]["'#{key}'"] = compile(node) + end + end + end + + js.object(mapped.each_with_object({} of String => String) do |(language, tokens), memo| + memo[language] = js.object(tokens) + end) + end + # -------------------------------------------------------------------------- # Wraps the application with the runtime @@ -271,6 +287,42 @@ module Mint const _PV = Symbol("Variable") const _PS = Symbol("Spread") + class Locale { + constructor(translations) { + this.locale = Object.keys(translations)[0]; + this.translations = translations; + this.listeners = new Set(); + } + + set(locale) { + if (this.locale != locale && this.translations[locale]) { + this.locale = locale; + + for (let listener of this.listeners) { + listener.forceUpdate(); + } + + return true + } else { + return false + } + } + + t(key) { + return this.translations[this.locale][key] + } + + _subscribe(owner) { + this.listeners.add(owner); + } + + _unsubscribe(owner) { + this.listeners.delete(owner); + } + } + + const _L = new Locale(#{compiled_locales}); + class RecordPattern { constructor(patterns) { this.patterns = patterns diff --git a/src/formatters/locale.cr b/src/formatters/locale.cr new file mode 100644 index 000000000..d2b899535 --- /dev/null +++ b/src/formatters/locale.cr @@ -0,0 +1,10 @@ +module Mint + class Formatter + def format(node : Ast::Locale) : String + body = + format node.fields + + "locale #{node.language} {\n#{indent(body.join(",\n"))}\n}" + end + end +end diff --git a/src/formatters/locale_key.cr b/src/formatters/locale_key.cr new file mode 100644 index 000000000..b7670c828 --- /dev/null +++ b/src/formatters/locale_key.cr @@ -0,0 +1,7 @@ +module Mint + class Formatter + def format(node : Ast::LocaleKey) : String + ":#{node.value}" + end + end +end diff --git a/src/formatters/top_level.cr b/src/formatters/top_level.cr index 00855929a..557ffb186 100644 --- a/src/formatters/top_level.cr +++ b/src/formatters/top_level.cr @@ -15,7 +15,8 @@ module Mint ast.stores + ast.suites + ast.enums + - ast.comments + ast.comments + + ast.locales ) .sort_by!(&.from) .map { |node| format node } diff --git a/src/messages/expected_end_of_file.cr b/src/messages/expected_end_of_file.cr index 7cfdb6c6d..de72a4289 100644 --- a/src/messages/expected_end_of_file.cr +++ b/src/messages/expected_end_of_file.cr @@ -14,6 +14,7 @@ message ExpectedEndOfFile do "Routes", "Store", "Suite", + "Locale", ] block do diff --git a/src/messages/locale_expected_closing_bracket.cr b/src/messages/locale_expected_closing_bracket.cr new file mode 100644 index 000000000..a6b94a825 --- /dev/null +++ b/src/messages/locale_expected_closing_bracket.cr @@ -0,0 +1,7 @@ +message LocaleExpectedClosingBracket do + title "Syntax Error" + + closing_bracket "locale", got + + snippet node +end diff --git a/src/messages/locale_expected_language.cr b/src/messages/locale_expected_language.cr new file mode 100644 index 000000000..93af3a972 --- /dev/null +++ b/src/messages/locale_expected_language.cr @@ -0,0 +1,13 @@ +message LocaleExpectedLanguage do + title "Syntax Error" + + block do + text "I was looking for the" + bold "language of a locale" + text "but found" + code got + text "instead." + end + + snippet node +end diff --git a/src/messages/locale_expected_opening_bracket.cr b/src/messages/locale_expected_opening_bracket.cr new file mode 100644 index 000000000..f109707cc --- /dev/null +++ b/src/messages/locale_expected_opening_bracket.cr @@ -0,0 +1,7 @@ +message LocaleExpectedOpeningBracket do + title "Syntax Error" + + opening_bracket "locale", got + + snippet node +end diff --git a/src/messages/translation_mismatch.cr b/src/messages/translation_mismatch.cr new file mode 100644 index 000000000..10db77b5f --- /dev/null +++ b/src/messages/translation_mismatch.cr @@ -0,0 +1,15 @@ +message TranslationMismatch do + title "Type Error" + + block do + text "The type of the key" + bold value + text "in the language:" + bold language + text "does not match the type in another language." + end + + was_expecting_type expected, got + + snippet node +end diff --git a/src/messages/translation_missing.cr b/src/messages/translation_missing.cr new file mode 100644 index 000000000..d22258ebd --- /dev/null +++ b/src/messages/translation_missing.cr @@ -0,0 +1,10 @@ +message TranslationMissing do + title "Type Error" + + block do + text "Translations are not specified for the key:" + bold value + end + + snippet node +end diff --git a/src/messages/translation_not_translated.cr b/src/messages/translation_not_translated.cr new file mode 100644 index 000000000..4c74f2d3c --- /dev/null +++ b/src/messages/translation_not_translated.cr @@ -0,0 +1,12 @@ +message TranslationNotTranslated do + title "Type Error" + + block do + text "There is no translation for the key:" + bold value + text "in the language:" + bold language + end + + snippet node +end diff --git a/src/parser.cr b/src/parser.cr index c2d9083a8..1b16dff1f 100644 --- a/src/parser.cr +++ b/src/parser.cr @@ -5,6 +5,7 @@ module Mint getter ast = Ast.new getter data : Ast::Data getter refs = [] of {Ast::Variable, Ast::HtmlComponent | Ast::HtmlElement} + getter locales = [] of Ast::LocaleKey getter position = 0 def initialize(input : String, @file) diff --git a/src/parsers/basic_expression.cr b/src/parsers/basic_expression.cr index 051489ac5..ffb07f358 100644 --- a/src/parsers/basic_expression.cr +++ b/src/parsers/basic_expression.cr @@ -9,6 +9,7 @@ module Mint asset_directive || inline_directive || env || + locale_key || here_doc || string_literal || regexp_literal || diff --git a/src/parsers/component.cr b/src/parsers/component.cr index 53b5bc289..7d2a6a841 100644 --- a/src/parsers/component.cr +++ b/src/parsers/component.cr @@ -17,7 +17,8 @@ module Mint name = type_id! ComponentExpectedName - # Clear refs here because it's on the parser + # Clear refs and locales here because it's on the parser + locales.clear refs.clear body = block( @@ -77,6 +78,7 @@ module Mint end self << Ast::Component.new( + locales: !locales.empty?, global: global || false, properties: properties, functions: functions, diff --git a/src/parsers/locale.cr b/src/parsers/locale.cr new file mode 100644 index 000000000..c58db9f58 --- /dev/null +++ b/src/parsers/locale.cr @@ -0,0 +1,39 @@ +module Mint + class Parser + syntax_error LocaleExpectedClosingBracket + syntax_error LocaleExpectedOpeningBracket + syntax_error LocaleExpectedLanguage + + def locale : Ast::Locale? + start do |start_position| + comment = self.comment + + next unless keyword "locale" + whitespace + + language = gather do + next unless char.ascii_lowercase? + chars { |char| char.ascii_letter? || char.ascii_number? } + end + + raise LocaleExpectedLanguage unless language + whitespace + + fields = block( + opening_bracket: LocaleExpectedOpeningBracket, + closing_bracket: LocaleExpectedClosingBracket + ) do + list(terminator: '}', separator: ',') { record_field } + end + + self << Ast::Locale.new( + from: start_position, + language: language, + comment: comment, + fields: fields, + to: position, + input: data) + end + end + end +end diff --git a/src/parsers/locale_key.cr b/src/parsers/locale_key.cr new file mode 100644 index 000000000..0173a5da5 --- /dev/null +++ b/src/parsers/locale_key.cr @@ -0,0 +1,25 @@ +module Mint + class Parser + def locale_key : Ast::LocaleKey? + start do |start_position| + next unless char! ':' + + value = gather do + next unless char.ascii_lowercase? + chars { |char| char.ascii_letter? || char.ascii_number? || char == '.' } + end + + next unless value + + Ast::LocaleKey.new( + from: start_position, + value: value, + to: position, + input: data).tap do |node| + locales << node + self << node + end + end + end + end +end diff --git a/src/parsers/top_level.cr b/src/parsers/top_level.cr index ca1a3b2d5..325bc2441 100644 --- a/src/parsers/top_level.cr +++ b/src/parsers/top_level.cr @@ -25,6 +25,7 @@ module Mint record_definition || self.enum || provider || + locale || routes || store || suite || @@ -51,6 +52,8 @@ module Mint @ast.enums << item when Ast::Comment @ast.comments << item + when Ast::Locale + @ast.locales << item end end end diff --git a/src/type_checker.cr b/src/type_checker.cr index 109ece493..9dc5e5ed6 100644 --- a/src/type_checker.cr +++ b/src/type_checker.cr @@ -43,7 +43,7 @@ module Mint delegate checked, record_field_lookup, component_records, to: artifacts delegate types, variables, ast, lookups, cache, to: artifacts - delegate assets, resolve_order, to: artifacts + delegate assets, resolve_order, locales, to: artifacts delegate component?, component, stateful?, current_top_level_entity?, to: scope delegate format, to: formatter @@ -54,6 +54,7 @@ module Mint @types = {} of String => Ast::Node @records = [] of Record @top_level_entity : Ast::Node? + @languages : Array(String) @referee : Ast::Node? @returns : Hash(Ast::Node, Array(Ast::ReturnCall)) = {} of Ast::Node => Array(Ast::ReturnCall) @@ -80,10 +81,13 @@ module Mint def initialize(ast : Ast, @check_env = true, @web_components = [] of String) ast.normalize + @languages = ast.unified_locales.map(&.language) @artifacts = Artifacts.new(ast) @scope = Scope.new(ast, records) resolve_records + + ast.unified_locales.each { |locale| check_locale(locale) } end def debug diff --git a/src/type_checker/artifacts.cr b/src/type_checker/artifacts.cr index 7ba74d1df..f825f86a9 100644 --- a/src/type_checker/artifacts.cr +++ b/src/type_checker/artifacts.cr @@ -2,7 +2,7 @@ module Mint class TypeChecker class Artifacts getter ast, lookups, cache, checked, record_field_lookup, assets - getter types, variables, component_records, resolve_order + getter types, variables, component_records, resolve_order, locales def initialize(@ast : Ast, @component_records = {} of Ast::Component => Record, @@ -12,6 +12,7 @@ module Mint @assets = [] of Ast::Directives::Asset, @types = {} of Ast::Node => Checkable, @cache = {} of Ast::Node => Checkable, + @locales = {} of String => Hash(String, Ast::Node), @resolve_order = [] of Ast::Node, @checked = Set(Ast::Node).new) end diff --git a/src/type_checker/scope.cr b/src/type_checker/scope.cr index 6b5340a74..1567e341d 100644 --- a/src/type_checker/scope.cr +++ b/src/type_checker/scope.cr @@ -193,6 +193,14 @@ module Mint ensure @levels = old_levels end + when Ast::LocaleKey + old_levels = @levels + @levels = [] of Node + begin + return yield + ensure + @levels = old_levels + end when Ast::Function, Ast::Get if store = @functions[node]? diff --git a/src/type_checkers/locale.cr b/src/type_checkers/locale.cr new file mode 100644 index 000000000..d8261c6f9 --- /dev/null +++ b/src/type_checkers/locale.cr @@ -0,0 +1,28 @@ +module Mint + class TypeChecker + def check_locale(node : Ast::Locale) + node.fields.each do |field| + check_locale_record(field, nil, node.language) + end + end + + def check_locale_record(node : Ast::RecordField, prefix : String?, language : String) + field_prefix = + if prefix + "#{prefix}.#{node.key.value}" + else + node.key.value + end + + case item = node.value + when Ast::Record + item.fields.each do |field| + check_locale_record(field, field_prefix, language) + end + else + locales[field_prefix] ||= {} of String => Ast::Node + locales[field_prefix][language] = item + end + end + end +end diff --git a/src/type_checkers/locale_key.cr b/src/type_checkers/locale_key.cr new file mode 100644 index 000000000..60990780d --- /dev/null +++ b/src/type_checkers/locale_key.cr @@ -0,0 +1,48 @@ +module Mint + class TypeChecker + type_error TranslationNotTranslated + type_error TranslationMismatch + type_error TranslationMissing + + def check(node : Ast::LocaleKey) : Checkable + raise TranslationMissing, { + "value" => node.value, + "node" => node, + } unless translations = locales[node.value]? + + scope(node) do + result = nil + + @languages.each do |language| + raise TranslationNotTranslated, { + "value" => node.value, + "language" => language, + "node" => node, + } unless value = translations[language]? + + type = + resolve(value) + + result = + if result + resolved = Comparer.compare(result, type) + + raise TranslationMismatch, { + "value" => node.value, + "language" => language, + "expected" => result, + "got" => type, + "node" => node, + } unless resolved + + resolved + else + type + end + end + + result.not_nil! + end + end + end +end From 6e51ae076ad442fb72e97ff5bd6ec42343b52ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 18 Jul 2023 16:49:27 +0200 Subject: [PATCH 15/19] Update operation.cr --- src/formatters/operation.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/formatters/operation.cr b/src/formatters/operation.cr index 36454b3d7..13d1eccf6 100644 --- a/src/formatters/operation.cr +++ b/src/formatters/operation.cr @@ -7,8 +7,7 @@ module Mint right = format node.right - if node.new_line? && - node.right.is_a?(Ast::Operation) + if node.new_line? "#{left} #{node.operator}\n#{indent(right.remove_all_leading_whitespace)}" else "#{left} #{node.operator} #{right}" From 92833ca8c448b6f653240151fdc0d8915d0445a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Fri, 21 Jul 2023 14:12:22 +0200 Subject: [PATCH 16/19] Update test. --- spec/formatters/operation_multi_line_with_pipe | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/formatters/operation_multi_line_with_pipe b/spec/formatters/operation_multi_line_with_pipe index 08ece47e3..cb23bbb07 100644 --- a/spec/formatters/operation_multi_line_with_pipe +++ b/spec/formatters/operation_multi_line_with_pipe @@ -16,6 +16,7 @@ module Test { fun test : Bool { ("Hello" - |> identity()) == "True" + |> identity()) == + "True" } } From 44ab595bb15bf3512c1460d56a217cde4b079a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 8 Aug 2023 14:22:01 +0200 Subject: [PATCH 17/19] Revert change to the operation formatting. --- spec/formatters/operation_multi_line_with_pipe | 3 +-- src/formatters/operation.cr | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/formatters/operation_multi_line_with_pipe b/spec/formatters/operation_multi_line_with_pipe index cb23bbb07..08ece47e3 100644 --- a/spec/formatters/operation_multi_line_with_pipe +++ b/spec/formatters/operation_multi_line_with_pipe @@ -16,7 +16,6 @@ module Test { fun test : Bool { ("Hello" - |> identity()) == - "True" + |> identity()) == "True" } } diff --git a/src/formatters/operation.cr b/src/formatters/operation.cr index 13d1eccf6..36454b3d7 100644 --- a/src/formatters/operation.cr +++ b/src/formatters/operation.cr @@ -7,7 +7,8 @@ module Mint right = format node.right - if node.new_line? + if node.new_line? && + node.right.is_a?(Ast::Operation) "#{left} #{node.operator}\n#{indent(right.remove_all_leading_whitespace)}" else "#{left} #{node.operator} #{right}" From a8ecd78a04385fb9f6b193ec4d28925364c03d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Tue, 8 Aug 2023 14:28:13 +0200 Subject: [PATCH 18/19] Update Locale.md --- documentation/Language/Locale.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/Language/Locale.md b/documentation/Language/Locale.md index 6c66d4446..8acc83614 100644 --- a/documentation/Language/Locale.md +++ b/documentation/Language/Locale.md @@ -45,3 +45,5 @@ Every translation is typed checked: * all translations of the same key must have the same type * locale keys must have translations in every defined language + +Locales are open like Modules are so they can be defined in multiple places. From 3c4010ca4ec7df22e96f8580ac6f6b982f4536b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szikszai=20Guszt=C3=A1v?= Date: Mon, 11 Sep 2023 09:54:34 +0200 Subject: [PATCH 19/19] Apply suggestions from code review Co-authored-by: jansul --- documentation/Language/Locale.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/documentation/Language/Locale.md b/documentation/Language/Locale.md index 8acc83614..e34884376 100644 --- a/documentation/Language/Locale.md +++ b/documentation/Language/Locale.md @@ -31,6 +31,16 @@ To localize a value you need to use the locale token: or if it's a function if can be called: ``` +locale en { + ui: { + buttons: { + ok: (param1 : String, param2 : String) { + "Button #{param1} #{param2}!" + } + } + } +} + :ui.buttons.ok(param1, param2) ```