From 09f47360199abdeed88cd644c3bfaab779c104a5 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 28 Oct 2022 10:49:34 -0300 Subject: [PATCH 01/26] pick up from #961 --- src/LanguageServer.jl | 1 + src/languageserverinstance.jl | 2 + src/protocol/initialize.jl | 2 + src/protocol/protocol.jl | 1 + src/protocol/semantic.jl | 125 ++++++++++++++++++++ src/requests/semantic.jl | 208 ++++++++++++++++++++++++++++++++++ test/requests/semantic.jl | 10 ++ 7 files changed, 349 insertions(+) create mode 100644 src/protocol/semantic.jl create mode 100644 src/requests/semantic.jl create mode 100644 test/requests/semantic.jl diff --git a/src/LanguageServer.jl b/src/LanguageServer.jl index 259fc01f..1e8dadb8 100644 --- a/src/LanguageServer.jl +++ b/src/LanguageServer.jl @@ -36,6 +36,7 @@ include("requests/actions.jl") include("requests/init.jl") include("requests/signatures.jl") include("requests/highlight.jl") +include("requests/semantic.jl") include("utilities.jl") end diff --git a/src/languageserverinstance.jl b/src/languageserverinstance.jl index 751e9e38..a88a553a 100644 --- a/src/languageserverinstance.jl +++ b/src/languageserverinstance.jl @@ -344,6 +344,8 @@ function Base.run(server::LanguageServerInstance) msg_dispatcher[textDocument_prepareRename_request_type] = request_wrapper(textDocument_prepareRename_request, server) msg_dispatcher[textDocument_documentSymbol_request_type] = request_wrapper(textDocument_documentSymbol_request, server) msg_dispatcher[textDocument_documentHighlight_request_type] = request_wrapper(textDocument_documentHighlight_request, server) + msg_dispatcher[textDocument_semanticTokens_request_type] = request_wrapper(textDocument_semanticTokens_request, server) + msg_dispatcher[textDocument_semanticTokens_full_request_type] = request_wrapper(textDocument_semanticTokens_full_request, server) msg_dispatcher[julia_getModuleAt_request_type] = request_wrapper(julia_getModuleAt_request, server) msg_dispatcher[julia_getDocAt_request_type] = request_wrapper(julia_getDocAt_request, server) msg_dispatcher[textDocument_hover_request_type] = request_wrapper(textDocument_hover_request, server) diff --git a/src/protocol/initialize.jl b/src/protocol/initialize.jl index fa4cc395..5b5ce04b 100644 --- a/src/protocol/initialize.jl +++ b/src/protocol/initialize.jl @@ -77,6 +77,7 @@ end publishDiagnostics::Union{PublishDiagnosticsClientCapabilities,Missing} foldingRange::Union{FoldingRangeClientCapabilities,Missing} selectionRange::Union{SelectionRangeClientCapabilities,Missing} + semanticTokens::Union{SemanticTokensClientCapabilities,Missing} end @dict_readable struct WindowClientCapabilities <: Outbound @@ -183,6 +184,7 @@ struct ServerCapabilities <: Outbound foldingRangeProvider::Union{Bool,FoldingRangeOptions,FoldingRangeRegistrationOptions,Missing} executeCommandProvider::Union{ExecuteCommandOptions,Missing} selectionRangeProvider::Union{Bool,SelectionRangeOptions,SelectionRangeRegistrationOptions,Missing} + semanticTokensProvider::Union{Bool,SemanticTokensOptions,SemanticTokensRegistrationOptions} workspaceSymbolProvider::Union{Bool,Missing} workspace::Union{WorkspaceOptions,Missing} experimental::Union{Any,Missing} diff --git a/src/protocol/protocol.jl b/src/protocol/protocol.jl index 5ce30e3c..bbdf3f76 100644 --- a/src/protocol/protocol.jl +++ b/src/protocol/protocol.jl @@ -5,6 +5,7 @@ include("formatting.jl") include("hover.jl") include("goto.jl") include("highlight.jl") +include("semantic.jl") include("signature.jl") include("symbols.jl") include("features.jl") diff --git a/src/protocol/semantic.jl b/src/protocol/semantic.jl new file mode 100644 index 00000000..f946d9b4 --- /dev/null +++ b/src/protocol/semantic.jl @@ -0,0 +1,125 @@ +# https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens +const SemanticTokenKind = String +const SemanticTokenKinds = ( + Struct="struct", + TypeParameter="typeParameter", + Parameter="parameter", + Variable="variable", + Property="property", + Function="function", + Macro="macro", + Keyword="keyword", + Comment="comment", + String="string", + Number="number", + Regexp="regexp", + Operator="operator" +) + +const SemanticTokenModifiersKind = String +const SemanticTokenModifiersKinds = ( + Declaration="declaration", + Definition="definition", + Modification="modification", + Documentation="documentation", + DefaultLibrary="defaultLibrary") + + +struct SemanticTokensLegend <: Outbound + tokenTypes::Vector{String} # The token types a server uses. + + tokenModifiers::Vector{String} # The token modifiers a server uses. +end + +const JuliaSemanticTokensLegend = SemanticTokensLegend( + collect(values(SemanticTokenKinds)), + collect(values(SemanticTokenModifiersKinds)) +) + +function semantic_token_encoding(token::String)::UInt32 + for (i, type) in enumerate(JuliaSemanticTokensLegend.tokenTypes) + if token == type + return i - 1 # -1 to shift to 0-based indexing + end + end +end + +@dict_readable struct SemanticTokensFullDelta <: Outbound + delta::Union{Bool,Missing} +end + +@dict_readable struct SemanticTokensClientCapabilitiesRequests <: Outbound + range::Union{Bool,Missing} + full::Union{Bool,Missing,SemanticTokensFullDelta} + +end +@dict_readable struct SemanticTokensClientCapabilities <: Outbound + dynamicRegistration::Union{Bool,Missing} + tokenTypes::Vector{String} + tokenModifiers::Vector{String} + formats::Vector{String} + overlappingTokenSupport::Union{Bool,Missing} + multilineTokenSupport::Union{Bool,Missing} +end + +struct SemanticTokensOptions <: Outbound + legend::SemanticTokensLegend + range::Union{Bool,Missing} + full::Union{Bool,SemanticTokensFullDelta,Missing} +end + +struct SemanticTokensRegistrationOptions <: Outbound + documentSelector::Union{DocumentSelector,Nothing} + # workDoneProgress::Union{Bool,Missing} +end + +@dict_readable struct SemanticTokensParams <: Outbound + textDocument::TextDocumentIdentifier + # position::Position + workDoneToken::Union{Int,String,Missing} # ProgressToken + partialResultToken::Union{Int,String,Missing} # ProgressToken +end + +struct SemanticTokens <: Outbound + resultId::Union{String,Missing} + data::Vector{UInt32} +end + +SemanticTokens(data::Vector{UInt32}) = SemanticTokens(missing, data) + + + +struct SemanticTokensPartialResult <: Outbound + data::Vector{UInt32} +end + +struct SemanticTokensDeltaParams <: Outbound + workDoneToken::Union{Int,String,Missing} + partialResultToken::Union{Int,String,Missing} # ProgressToken + textDocument::TextDocumentIdentifier + previousResultId::String +end +struct SemanticTokensEdit <: Outbound + start::UInt32 + deleteCount::Int + data::Union{Vector{Int},Missing} +end +struct SemanticTokensDelta <: Outbound + resultId::Union{String,Missing} + edits::Vector{SemanticTokensEdit} +end + +struct SemanticTokensDeltaPartialResult <: Outbound + edits::Vector{SemanticTokensEdit} +end + +struct SemanticTokensRangeParams <: Outbound + workDoneToken::Union{Int,String,Missing} + partialResultToken::Union{Int,String,Missing} # ProgressToken + textDocument::TextDocumentIdentifier + range::Range +end + +struct SemanticTokensWorkspaceClientCapabilities <: Outbound + refreshSupport::Union{Bool,Missing} +end diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl new file mode 100644 index 00000000..91007099 --- /dev/null +++ b/src/requests/semantic.jl @@ -0,0 +1,208 @@ +function textDocument_semanticTokens_request(params::SemanticTokensParams, server::LanguageServerInstance, conn) + doc = getdocument(server, URI2(params.textDocument.uri)) + offset = get_offset(doc, params.position) + identifier = get_identifier(getcst(doc), offset) + identifier !== nothing || return nothing + highlights = DocumentHighlight[] + for_each_ref(identifier) do ref, doc1, o + if doc1._uri == doc._uri + kind = StaticLint.hasbinding(ref) ? DocumentHighlightKinds.Write : DocumentHighlightKinds.Read + push!(highlights, DocumentHighlight(Range(doc, o .+ (0:ref.span)), kind)) + end + end + return isempty(highlights) ? nothing : highlights +end + +struct SemanticToken + # token line number, relative to the previous token + deltaLine::UInt32 + # token start character, relative to the previous token + # (relative to 0 or the previous token’s start if they are on the same line) + deltaStart::UInt32 + # the length of the token. + length::UInt32 + # will be looked up in SemanticTokensLegend.tokenTypes + tokenType::UInt32 + # each set bit will be looked up in SemanticTokensLegend.tokenModifiers + tokenModifiers::UInt32 +end + +function SemanticToken(deltaLine::UInt32, + deltaStart::UInt32, + length::UInt32, + tokenType::String, + tokenModifiers::String) + # TODO look up int encodings for tokenType and tokenModifiers + SemanticToken( + deltaLine, + deltaStart, + length, + semantic_token_encoding(tokenType), + 0 # TODO look up int encodings for tokenType and tokenModifiers + ) +end + +# function SemanticToken(ex::EXPR) + +# end + + +function semantic_tokens(tokens)::SemanticTokens + token_vectors = map(tokens) do token::SemanticToken + # token_index = i - 1 + [ + token.deltaLine, + token.deltaStart, + token.length, + token.tokenType, + token.tokenModifiers + ] + end + SemanticTokens(Iterators.flatten(token_vectors) |> collect) +end + +function textDocument_semanticTokens_full_request(params::SemanticTokensParams, + server::LanguageServerInstance, conn)::Union{SemanticTokens,Nothing} + uri = params.textDocument.uri + doc = getdocument(server, URI2(uri)) + ts = collect(every_semantic_token(doc)) + return semantic_tokens(ts) +end + +@doc """ +Iterator interface for providing tokens from a Document + +parse applies these types + Parser.SyntaxNode + Parser.EXPR + Parser.INSTANCE + Parser.HEAD{K} + Parser.IDENTIFIER + Parser.KEYWORD{K} + Parser.LITERAL{K} + Parser.OPERATOR{P,K,dot} + Parser.PUNCTUATION + Parser.QUOTENODE + +┌ Info: 1:60 file( new scope lint ) +│ 1:60 function( Binding(main:: (1 refs)) new scope) +│ 1:8 call( ) +│ 1:4 main * +│ 9:48 block( ) +│ 9:28 1:2 OP: =( ) +│ 9:10 s Binding(s:: (3 refs)) * +│ 11:26 STRING: hello world! +│ 29:40 call( ) +│ 29:35 println * +│ 36:36 s * +│ 41:48 macrocall( ) +│ 41:46 @show * +│ 47:46 NOTHING: nothing +└ 47:48 s * + """ + +function expression_to_maybe_token(ex::EXPR, offset) + kind = semantic_token_kind(ex) + if kind === nothing + return nothing + end + name = C.get_name(ex) + name_offset = 0 + # get the offset of the name expr + if name !== nothing + found = false + for x in ex + if x == name + found = true + break + end + name_offset += x.fullspan + end + if !found + name_offset = -1 + end + end + line, char = get_offset(doc, offset) + return SemanticToken( + line, + char, + ex.span, + semantic_token_encoding(kind), + 0 + ) +end +function every_expression_with_offset(expr::EXPR, offset=0) + every_expression = Tuple{EXPR,Int64}[] + for ex in expr + push!((ex, offset), every_expression) + if !isempty(ex) + push!((ex, offset), every_expression_with_offset(ex, offset)...) + end + offset += ex.fullspan + end +end +function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc) + kind = semantic_token_kind(ex) + if kind === nothing + return nothing + end + name = C.get_name(ex) + name_offset = 0 + # get the offset of the name expr + if name !== nothing + found = false + for x in ex + if x == name + found = true + break + end + name_offset += x.fullspan + end + if !found + name_offset = -1 + end + end + line, char = get_offset(doc, offset) + return SemanticToken( + line, + char, + ex.span, + semantic_token_encoding(kind), + 0 + ) +end + +function every_semantic_token(doc) + root_expr = getcst(doc) + maybe_tokens = map(expr_offset_to_maybe_token, + map((ex, offset) -> (ex, offset, doc), + every_expression_with_offset(root_expr))) + filter(maybe_token -> maybe_token !== nothing, maybe_tokens) +end + + +""" +Get the semantic token kind for `expr`, which is assumed to be an identifier +""" +function semantic_token_kind(expr::EXPR)::Union{String,Nothing} + # C.isidentifier(expr) || return nothing + return if C.defines_function(expr)# || C.is_func_call(expr) + SemanticTokenKinds.Function + elseif C.defines_struct(expr) + SemanticTokenKinds.Struct + elseif C.defines_macro(expr) + SemanticTokenKinds.Macro + # elseif C.isoperator(expr) + # SemanticTokenKinds.Operator + end +end +const C = CSTParser + +span(token::SemanticToken) = token.length +function span(tokens::Vector{SemanticToken}) + lengths = map(tokens) do token + token.length + end + + reduce(+, lengths) +end diff --git a/test/requests/semantic.jl b/test/requests/semantic.jl new file mode 100644 index 00000000..68a964b7 --- /dev/null +++ b/test/requests/semantic.jl @@ -0,0 +1,10 @@ +semantic_token_test() = LanguageServer.textDocument_semanticTokens_full_request(LanguageServer.SemanticTokensParams(LanguageServer.TextDocumentIdentifier("testdoc"), missing, missing), server, server.jr_endpoint) + +@testset "function calls" begin + settestdoc(""" + function hello() + println("hello world") + end + """) + @test semantic_token_test() == SemanticToken(0, 9, 5, SemanticTokenKinds.Function) +end From 086bd0d0b4d7e5f43abeaab524f58bac94dd2ef8 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 28 Oct 2022 16:11:08 -0300 Subject: [PATCH 02/26] update code to match new signature --- src/protocol/semantic.jl | 5 +++-- src/requests/init.jl | 5 +++++ test/test_shared_init_request.jl | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/protocol/semantic.jl b/src/protocol/semantic.jl index f946d9b4..6495d659 100644 --- a/src/protocol/semantic.jl +++ b/src/protocol/semantic.jl @@ -13,7 +13,7 @@ const SemanticTokenKinds = ( String="string", Number="number", Regexp="regexp", - Operator="operator" + Operator="operator", ) const SemanticTokenModifiersKind = String @@ -22,7 +22,8 @@ const SemanticTokenModifiersKinds = ( Definition="definition", Modification="modification", Documentation="documentation", - DefaultLibrary="defaultLibrary") + DefaultLibrary="defaultLibrary", +) struct SemanticTokensLegend <: Outbound diff --git a/src/requests/init.jl b/src/requests/init.jl index a8b21f77..e4960596 100644 --- a/src/requests/init.jl +++ b/src/requests/init.jl @@ -30,6 +30,11 @@ function ServerCapabilities(client::ClientCapabilities) false, ExecuteCommandOptions(missing, collect(keys(LSActions))), true, + SemanticTokensOptions( + SemanticTokensLegend([values(SemanticTokenKinds)...], + [values(SemanticTokenModifiersKinds)...]), + true, + true), true, WorkspaceOptions(WorkspaceFoldersOptions(true, true)), missing diff --git a/test/test_shared_init_request.jl b/test/test_shared_init_request.jl index 358df0d9..d98d8b24 100644 --- a/test/test_shared_init_request.jl +++ b/test/test_shared_init_request.jl @@ -45,6 +45,7 @@ init_request = LanguageServer.InitializeParams( missing ), "off", + missing, # SemanticTokensClientCapabilities() missing, missing ) From 9627733649620668449f9abb67d9749d06cc154d Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 2 Nov 2022 10:11:17 -0300 Subject: [PATCH 03/26] cleanup --- src/requests/semantic.jl | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 91007099..edd0c52d 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -32,21 +32,16 @@ function SemanticToken(deltaLine::UInt32, length::UInt32, tokenType::String, tokenModifiers::String) - # TODO look up int encodings for tokenType and tokenModifiers + # FIXME look up int encodings for tokenType and tokenModifiers SemanticToken( deltaLine, deltaStart, length, semantic_token_encoding(tokenType), - 0 # TODO look up int encodings for tokenType and tokenModifiers + 0 # FIXME look up int encodings for tokenType and tokenModifiers ) end -# function SemanticToken(ex::EXPR) - -# end - - function semantic_tokens(tokens)::SemanticTokens token_vectors = map(tokens) do token::SemanticToken # token_index = i - 1 @@ -100,7 +95,6 @@ parse applies these types │ 47:46 NOTHING: nothing └ 47:48 s * """ - function expression_to_maybe_token(ex::EXPR, offset) kind = semantic_token_kind(ex) if kind === nothing From 3809be9ae33cd37f7d9ebf1dac5b8153cf9e32f5 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 2 Nov 2022 16:23:58 -0300 Subject: [PATCH 04/26] correct argument position --- test/test_shared_init_request.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_shared_init_request.jl b/test/test_shared_init_request.jl index d98d8b24..e86e73cc 100644 --- a/test/test_shared_init_request.jl +++ b/test/test_shared_init_request.jl @@ -40,12 +40,12 @@ init_request = LanguageServer.InitializeParams( missing, # PublishDiagnosticsClientCapabilities(), missing, # FoldingRangeClientCapabilities(), missing, # SelectionRangeClientCapabilities() + missing, # SemanticTokensClientCapabilities() ), missing, missing ), "off", - missing, # SemanticTokensClientCapabilities() missing, missing ) From 50f2c71d745a02c94e0c0668f9de0aa3876dea08 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 25 Nov 2022 14:23:11 -0300 Subject: [PATCH 05/26] correct push! --- src/requests/semantic.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index edd0c52d..6b004f4e 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -128,9 +128,10 @@ end function every_expression_with_offset(expr::EXPR, offset=0) every_expression = Tuple{EXPR,Int64}[] for ex in expr - push!((ex, offset), every_expression) + push!(every_expression, (ex, offset)) if !isempty(ex) - push!((ex, offset), every_expression_with_offset(ex, offset)...) + sub_expressions = every_expression_with_offset(ex, offset) + push!(every_expression, sub_expressions...) end offset += ex.fullspan end From ae6e470dc71d943408bca5462e4450d177b07182 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 25 Nov 2022 14:26:51 -0300 Subject: [PATCH 06/26] correct test file name (?) --- test/requests/{semantic.jl => test_semantic.jl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/requests/{semantic.jl => test_semantic.jl} (100%) diff --git a/test/requests/semantic.jl b/test/requests/test_semantic.jl similarity index 100% rename from test/requests/semantic.jl rename to test/requests/test_semantic.jl From ee6ddec08834c39ae7f6b8778f1fa6265533b1bd Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Mon, 28 Nov 2022 12:22:42 -0300 Subject: [PATCH 07/26] missing const with url --- src/protocol/messagedefs.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/protocol/messagedefs.jl b/src/protocol/messagedefs.jl index 5af22e93..894e0b1f 100644 --- a/src/protocol/messagedefs.jl +++ b/src/protocol/messagedefs.jl @@ -10,6 +10,7 @@ const textDocument_prepareRename_request_type = JSONRPC.RequestType("textDocumen const textDocument_documentSymbol_request_type = JSONRPC.RequestType("textDocument/documentSymbol", DocumentSymbolParams, Union{Vector{DocumentSymbol}, Vector{SymbolInformation}, Nothing}) const textDocument_documentHighlight_request_type = JSONRPC.RequestType("textDocument/documentHighlight", DocumentHighlightParams, Union{Vector{DocumentHighlight}, Nothing}) const textDocument_hover_request_type = JSONRPC.RequestType("textDocument/hover", TextDocumentPositionParams, Union{Hover, Nothing}) +const textDocument_semanticTokens_full_request_type = JSONRPC.RequestType("textDocument/semanticTokens/full", SemanticTokensParams, Union{SemanticTokens,Nothing}) const textDocument_didOpen_notification_type = JSONRPC.NotificationType("textDocument/didOpen", DidOpenTextDocumentParams) const textDocument_didClose_notification_type = JSONRPC.NotificationType("textDocument/didClose", DidCloseTextDocumentParams) const textDocument_didChange_notification_type = JSONRPC.NotificationType("textDocument/didChange", DidChangeTextDocumentParams) From 0ef4590720b853f24406c3bf336e8d70cfb5402a Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Mon, 28 Nov 2022 12:23:02 -0300 Subject: [PATCH 08/26] dunno what this was, but not implementing --- src/languageserverinstance.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/languageserverinstance.jl b/src/languageserverinstance.jl index a88a553a..207b1cff 100644 --- a/src/languageserverinstance.jl +++ b/src/languageserverinstance.jl @@ -344,7 +344,6 @@ function Base.run(server::LanguageServerInstance) msg_dispatcher[textDocument_prepareRename_request_type] = request_wrapper(textDocument_prepareRename_request, server) msg_dispatcher[textDocument_documentSymbol_request_type] = request_wrapper(textDocument_documentSymbol_request, server) msg_dispatcher[textDocument_documentHighlight_request_type] = request_wrapper(textDocument_documentHighlight_request, server) - msg_dispatcher[textDocument_semanticTokens_request_type] = request_wrapper(textDocument_semanticTokens_request, server) msg_dispatcher[textDocument_semanticTokens_full_request_type] = request_wrapper(textDocument_semanticTokens_full_request, server) msg_dispatcher[julia_getModuleAt_request_type] = request_wrapper(julia_getModuleAt_request, server) msg_dispatcher[julia_getDocAt_request_type] = request_wrapper(julia_getDocAt_request, server) From 512d8c982b0d489db73d43192828e738f86822f2 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Mon, 28 Nov 2022 12:23:39 -0300 Subject: [PATCH 09/26] bugfix, cleanup --- src/protocol/semantic.jl | 1 - src/requests/semantic.jl | 101 +++++++-------------------------------- 2 files changed, 16 insertions(+), 86 deletions(-) diff --git a/src/protocol/semantic.jl b/src/protocol/semantic.jl index 6495d659..ad5b856c 100644 --- a/src/protocol/semantic.jl +++ b/src/protocol/semantic.jl @@ -89,7 +89,6 @@ end SemanticTokens(data::Vector{UInt32}) = SemanticTokens(missing, data) - struct SemanticTokensPartialResult <: Outbound data::Vector{UInt32} end diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 6b004f4e..d3883307 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -1,18 +1,3 @@ -function textDocument_semanticTokens_request(params::SemanticTokensParams, server::LanguageServerInstance, conn) - doc = getdocument(server, URI2(params.textDocument.uri)) - offset = get_offset(doc, params.position) - identifier = get_identifier(getcst(doc), offset) - identifier !== nothing || return nothing - highlights = DocumentHighlight[] - for_each_ref(identifier) do ref, doc1, o - if doc1._uri == doc._uri - kind = StaticLint.hasbinding(ref) ? DocumentHighlightKinds.Write : DocumentHighlightKinds.Read - push!(highlights, DocumentHighlight(Range(doc, o .+ (0:ref.span)), kind)) - end - end - return isempty(highlights) ? nothing : highlights -end - struct SemanticToken # token line number, relative to the previous token deltaLine::UInt32 @@ -43,9 +28,9 @@ function SemanticToken(deltaLine::UInt32, end function semantic_tokens(tokens)::SemanticTokens - token_vectors = map(tokens) do token::SemanticToken - # token_index = i - 1 - [ + token_data = Vector{UInt32}(undef, length(tokens) * 4) + for (i_token, token) ∈ enumerate(tokens) + token_data[i_token:i_token+4] = [ token.deltaLine, token.deltaStart, token.length, @@ -53,78 +38,17 @@ function semantic_tokens(tokens)::SemanticTokens token.tokenModifiers ] end - SemanticTokens(Iterators.flatten(token_vectors) |> collect) + SemanticTokens(token_data) end function textDocument_semanticTokens_full_request(params::SemanticTokensParams, server::LanguageServerInstance, conn)::Union{SemanticTokens,Nothing} uri = params.textDocument.uri - doc = getdocument(server, URI2(uri)) + doc = getdocument(server, uri) ts = collect(every_semantic_token(doc)) return semantic_tokens(ts) end -@doc """ -Iterator interface for providing tokens from a Document - -parse applies these types - Parser.SyntaxNode - Parser.EXPR - Parser.INSTANCE - Parser.HEAD{K} - Parser.IDENTIFIER - Parser.KEYWORD{K} - Parser.LITERAL{K} - Parser.OPERATOR{P,K,dot} - Parser.PUNCTUATION - Parser.QUOTENODE - -┌ Info: 1:60 file( new scope lint ) -│ 1:60 function( Binding(main:: (1 refs)) new scope) -│ 1:8 call( ) -│ 1:4 main * -│ 9:48 block( ) -│ 9:28 1:2 OP: =( ) -│ 9:10 s Binding(s:: (3 refs)) * -│ 11:26 STRING: hello world! -│ 29:40 call( ) -│ 29:35 println * -│ 36:36 s * -│ 41:48 macrocall( ) -│ 41:46 @show * -│ 47:46 NOTHING: nothing -└ 47:48 s * - """ -function expression_to_maybe_token(ex::EXPR, offset) - kind = semantic_token_kind(ex) - if kind === nothing - return nothing - end - name = C.get_name(ex) - name_offset = 0 - # get the offset of the name expr - if name !== nothing - found = false - for x in ex - if x == name - found = true - break - end - name_offset += x.fullspan - end - if !found - name_offset = -1 - end - end - line, char = get_offset(doc, offset) - return SemanticToken( - line, - char, - ex.span, - semantic_token_encoding(kind), - 0 - ) -end function every_expression_with_offset(expr::EXPR, offset=0) every_expression = Tuple{EXPR,Int64}[] for ex in expr @@ -135,7 +59,9 @@ function every_expression_with_offset(expr::EXPR, offset=0) end offset += ex.fullspan end + every_expression end + function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc) kind = semantic_token_kind(ex) if kind === nothing @@ -157,7 +83,7 @@ function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc) name_offset = -1 end end - line, char = get_offset(doc, offset) + line, char = get_position_from_offset(doc, offset) return SemanticToken( line, char, @@ -169,9 +95,14 @@ end function every_semantic_token(doc) root_expr = getcst(doc) - maybe_tokens = map(expr_offset_to_maybe_token, - map((ex, offset) -> (ex, offset, doc), - every_expression_with_offset(root_expr))) + expressions = every_expression_with_offset(root_expr) + expressions_with_offsets = map(_tuple -> begin + ex::EXPR, offset::Int64 = _tuple + (ex, offset, doc) + end, expressions) + maybe_tokens = map(_tuple -> begin + expr_offset_to_maybe_token(_tuple...) + end, expressions_with_offsets) filter(maybe_token -> maybe_token !== nothing, maybe_tokens) end From 0023fb9cae212f40c86d82695cce6c386b700f6a Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Mon, 28 Nov 2022 18:11:52 -0300 Subject: [PATCH 10/26] disable ranged semantic tokens capabilities leave it for the next implementation / separate PR --- src/requests/init.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requests/init.jl b/src/requests/init.jl index e4960596..95e2a4f2 100644 --- a/src/requests/init.jl +++ b/src/requests/init.jl @@ -33,7 +33,7 @@ function ServerCapabilities(client::ClientCapabilities) SemanticTokensOptions( SemanticTokensLegend([values(SemanticTokenKinds)...], [values(SemanticTokenModifiersKinds)...]), - true, + false, true), true, WorkspaceOptions(WorkspaceFoldersOptions(true, true)), From 31102c049a255aa7b2b24383cc314926854e0361 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 30 Nov 2022 13:09:44 -0300 Subject: [PATCH 11/26] correct iteration ranges --- src/requests/semantic.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index d3883307..bda8d45a 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -28,8 +28,9 @@ function SemanticToken(deltaLine::UInt32, end function semantic_tokens(tokens)::SemanticTokens - token_data = Vector{UInt32}(undef, length(tokens) * 4) - for (i_token, token) ∈ enumerate(tokens) + token_data_size = length(tokens) * 5 + token_data = Vector{UInt32}(undef, token_data_size) + for (i_token, token::SemanticToken) ∈ zip(1:5:token_data_size, tokens) token_data[i_token:i_token+4] = [ token.deltaLine, token.deltaStart, From 6ed8071ab9f01fa9eeaa1e8f3431d9fbb9683c18 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 30 Nov 2022 13:10:49 -0300 Subject: [PATCH 12/26] sprinkle types --- src/requests/semantic.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index bda8d45a..a6380ff3 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -46,7 +46,7 @@ function textDocument_semanticTokens_full_request(params::SemanticTokensParams, server::LanguageServerInstance, conn)::Union{SemanticTokens,Nothing} uri = params.textDocument.uri doc = getdocument(server, uri) - ts = collect(every_semantic_token(doc)) + ts = collect(SemanticToken, every_semantic_token(doc)) return semantic_tokens(ts) end @@ -63,7 +63,7 @@ function every_expression_with_offset(expr::EXPR, offset=0) every_expression end -function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc) +function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc)::Union{Nothing,SemanticToken} kind = semantic_token_kind(ex) if kind === nothing return nothing From 656be23d7421b5a862ca8efbc0e61d15e3dcbcdb Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 30 Nov 2022 13:11:15 -0300 Subject: [PATCH 13/26] correct tuple handling --- src/requests/semantic.jl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index a6380ff3..85ca70bb 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -96,13 +96,10 @@ end function every_semantic_token(doc) root_expr = getcst(doc) - expressions = every_expression_with_offset(root_expr) - expressions_with_offsets = map(_tuple -> begin - ex::EXPR, offset::Int64 = _tuple - (ex, offset, doc) - end, expressions) + expressions_with_offsets = every_expression_with_offset(root_expr) maybe_tokens = map(_tuple -> begin - expr_offset_to_maybe_token(_tuple...) + ex::EXPR, offset::Int64 = _tuple + expr_offset_to_maybe_token(ex, offset, doc) end, expressions_with_offsets) filter(maybe_token -> maybe_token !== nothing, maybe_tokens) end From 15c87fb7a2f4210390cf0f52c3356150bfcf5beb Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 30 Nov 2022 13:11:32 -0300 Subject: [PATCH 14/26] address linter warning (unused arg) --- src/requests/semantic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 85ca70bb..64ece937 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -43,7 +43,7 @@ function semantic_tokens(tokens)::SemanticTokens end function textDocument_semanticTokens_full_request(params::SemanticTokensParams, - server::LanguageServerInstance, conn)::Union{SemanticTokens,Nothing} + server::LanguageServerInstance, _)::Union{SemanticTokens,Nothing} uri = params.textDocument.uri doc = getdocument(server, uri) ts = collect(SemanticToken, every_semantic_token(doc)) From ddc4086d8db197512fb4b592bb6be11aefcf7208 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 30 Nov 2022 13:12:34 -0300 Subject: [PATCH 15/26] match more token types --- src/requests/semantic.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 64ece937..763557e0 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -107,10 +107,21 @@ end """ Get the semantic token kind for `expr`, which is assumed to be an identifier + +See CSTParser.jl/src/interface.jl """ function semantic_token_kind(expr::EXPR)::Union{String,Nothing} # C.isidentifier(expr) || return nothing - return if C.defines_function(expr)# || C.is_func_call(expr) + + return if C.isidentifier(expr) + SemanticTokenKinds.Variable + elseif C.isoperator(expr) + SemanticTokenKinds.Operator + elseif C.isstringliteral(expr) || C.isstring(expr) + SemanticTokenKinds.String + elseif C.iskeyword(expr) + SemanticTokenKinds.Keyword + elseif C.defines_function(expr) # || C.is_func_call(expr) SemanticTokenKinds.Function elseif C.defines_struct(expr) SemanticTokenKinds.Struct From 91d42979a34be8c5e73bed2ffff616b498d7b5ce Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Thu, 1 Dec 2022 11:25:08 -0300 Subject: [PATCH 16/26] re-organize code using visitors --- src/requests/semantic.jl | 56 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 763557e0..e16c2d86 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -45,25 +45,12 @@ end function textDocument_semanticTokens_full_request(params::SemanticTokensParams, server::LanguageServerInstance, _)::Union{SemanticTokens,Nothing} uri = params.textDocument.uri - doc = getdocument(server, uri) - ts = collect(SemanticToken, every_semantic_token(doc)) + d = getdocument(server, uri) + ts = collect(SemanticToken, every_semantic_token(d)) return semantic_tokens(ts) end -function every_expression_with_offset(expr::EXPR, offset=0) - every_expression = Tuple{EXPR,Int64}[] - for ex in expr - push!(every_expression, (ex, offset)) - if !isempty(ex) - sub_expressions = every_expression_with_offset(ex, offset) - push!(every_expression, sub_expressions...) - end - offset += ex.fullspan - end - every_expression -end - -function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc)::Union{Nothing,SemanticToken} +function expr_offset_to_maybe_token(ex::EXPR, offset::Integer, document::Document)::Union{Nothing,SemanticToken} kind = semantic_token_kind(ex) if kind === nothing return nothing @@ -84,7 +71,7 @@ function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc)::Union{Nothing name_offset = -1 end end - line, char = get_position_from_offset(doc, offset) + line, char = get_position_from_offset(document, offset) return SemanticToken( line, char, @@ -94,14 +81,33 @@ function expr_offset_to_maybe_token(ex::EXPR, offset::Int64, doc)::Union{Nothing ) end -function every_semantic_token(doc) - root_expr = getcst(doc) - expressions_with_offsets = every_expression_with_offset(root_expr) - maybe_tokens = map(_tuple -> begin - ex::EXPR, offset::Int64 = _tuple - expr_offset_to_maybe_token(ex, offset, doc) - end, expressions_with_offsets) - filter(maybe_token -> maybe_token !== nothing, maybe_tokens) +mutable struct ExpressionVisitorState + collected_tokens::Vector{SemanticToken} + document::Document +end +ExpressionVisitorState(d::Document) = ExpressionVisitorState(SemanticToken[], d) + +function visit_every_expression_with_offset(expr_in::EXPR, state::ExpressionVisitorState, offset::Integer=0)::Nothing + for e ∈ expr_in + # ( maybe ) collect this expression + maybe_token = expr_offset_to_maybe_token(e, offset, state.document) + if maybe_token !== nothing + push!(state.collected_tokens, maybe_token) + end + + # recurse into e's subtrees + if !isempty(e) + visit_every_expression_with_offset(e, state, offset) + end + offset += e.fullspan + end +end + +function every_semantic_token(document::Document) + root_expr = getcst(document) + state = ExpressionVisitorState(document) + visit_every_expression_with_offset(root_expr, state) + state.collected_tokens end From f02f744792b8aa123a1519ad953554736eb8259a Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Thu, 1 Dec 2022 11:38:52 -0300 Subject: [PATCH 17/26] it seems tests aren't running correctly? --- test/requests/test_semantic.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/requests/test_semantic.jl b/test/requests/test_semantic.jl index 68a964b7..9696ed91 100644 --- a/test/requests/test_semantic.jl +++ b/test/requests/test_semantic.jl @@ -8,3 +8,7 @@ semantic_token_test() = LanguageServer.textDocument_semanticTokens_full_request( """) @test semantic_token_test() == SemanticToken(0, 9, 5, SemanticTokenKinds.Function) end + +@testset "add more, testing if running ok" begin + @test false == true +end From 13386d825c857aacaed7f41693a4f8ccacf8125a Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Thu, 1 Dec 2022 12:59:33 -0300 Subject: [PATCH 18/26] slight cleanup --- src/requests/semantic.jl | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index e16c2d86..136b4405 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -1,3 +1,5 @@ +const C = CSTParser + struct SemanticToken # token line number, relative to the previous token deltaLine::UInt32 @@ -117,7 +119,6 @@ Get the semantic token kind for `expr`, which is assumed to be an identifier See CSTParser.jl/src/interface.jl """ function semantic_token_kind(expr::EXPR)::Union{String,Nothing} - # C.isidentifier(expr) || return nothing return if C.isidentifier(expr) SemanticTokenKinds.Variable @@ -127,23 +128,13 @@ function semantic_token_kind(expr::EXPR)::Union{String,Nothing} SemanticTokenKinds.String elseif C.iskeyword(expr) SemanticTokenKinds.Keyword - elseif C.defines_function(expr) # || C.is_func_call(expr) + elseif C.defines_function(expr) SemanticTokenKinds.Function elseif C.defines_struct(expr) SemanticTokenKinds.Struct elseif C.defines_macro(expr) SemanticTokenKinds.Macro - # elseif C.isoperator(expr) - # SemanticTokenKinds.Operator + elseif C.isnumber(expr) + SemanticTokenKinds.Number end end -const C = CSTParser - -span(token::SemanticToken) = token.length -function span(tokens::Vector{SemanticToken}) - lengths = map(tokens) do token - token.length - end - - reduce(+, lengths) -end From 950e5deb6df1a7769d9316ed5ec3eb47c08ef4e1 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Thu, 1 Dec 2022 12:59:58 -0300 Subject: [PATCH 19/26] correct test code (still WIP) --- test/requests/test_semantic.jl | 19 ++++++++----------- test/test_shared_server.jl | 1 + 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/test/requests/test_semantic.jl b/test/requests/test_semantic.jl index 9696ed91..d24f8cd7 100644 --- a/test/requests/test_semantic.jl +++ b/test/requests/test_semantic.jl @@ -1,14 +1,11 @@ -semantic_token_test() = LanguageServer.textDocument_semanticTokens_full_request(LanguageServer.SemanticTokensParams(LanguageServer.TextDocumentIdentifier("testdoc"), missing, missing), server, server.jr_endpoint) -@testset "function calls" begin - settestdoc(""" - function hello() - println("hello world") - end - """) - @test semantic_token_test() == SemanticToken(0, 9, 5, SemanticTokenKinds.Function) -end +@testitem "simple token" begin + include("../test_shared_server.jl") -@testset "add more, testing if running ok" begin - @test false == true + doc = settestdoc(""" +a=1 + """) + let _LS = LanguageServer + @test token_full_test() == _LS.SemanticTokens(UInt32[0, 0, 9, 5, _LS.semantic_token_encoding(_LS.SemanticTokenKinds.Variable)]) + end end diff --git a/test/test_shared_server.jl b/test/test_shared_server.jl index 11aeb472..486d9e14 100644 --- a/test/test_shared_server.jl +++ b/test/test_shared_server.jl @@ -21,6 +21,7 @@ ref_test(line, char) = LanguageServer.textDocument_references_request(LanguageSe rename_test(line, char) = LanguageServer.textDocument_rename_request(LanguageServer.RenameParams(LanguageServer.TextDocumentIdentifier(uri"untitled:testdoc"), LanguageServer.Position(line, char), missing, "newname"), server, server.jr_endpoint) hover_test(line, char) = LanguageServer.textDocument_hover_request(LanguageServer.TextDocumentPositionParams(LanguageServer.TextDocumentIdentifier(uri"untitled:testdoc"), LanguageServer.Position(line, char)), server, server.jr_endpoint) range_formatting_test(line0, char0, line1, char1) = LanguageServer.textDocument_range_formatting_request(LanguageServer.DocumentRangeFormattingParams(LanguageServer.TextDocumentIdentifier(uri"untitled:testdoc"), LanguageServer.Range(LanguageServer.Position(line0, char0), LanguageServer.Position(line1, char1)), LanguageServer.FormattingOptions(4, true, missing, missing, missing)), server, server.jr_endpoint) +token_full_test() = LanguageServer.textDocument_semanticTokens_full_request(LanguageServer.SemanticTokensParams(LanguageServer.TextDocumentIdentifier(uri"untitled:testdoc"), missing, missing), server, server.jr_endpoint) # TODO Replace this with a proper mock endpoint JSONRPC.send(::Nothing, ::Any, ::Any) = nothing From 1563c7195be8a7dafd19db665dd33f6e003928c1 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Thu, 1 Dec 2022 17:36:35 -0300 Subject: [PATCH 20/26] track doc, offset in state, also carry ext_env for semantics --- src/requests/semantic.jl | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 136b4405..af934c32 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -48,12 +48,15 @@ function textDocument_semanticTokens_full_request(params::SemanticTokensParams, server::LanguageServerInstance, _)::Union{SemanticTokens,Nothing} uri = params.textDocument.uri d = getdocument(server, uri) - ts = collect(SemanticToken, every_semantic_token(d)) + + external_env = getenv(d, server) + + ts = collect(SemanticToken, every_semantic_token(d, external_env)) return semantic_tokens(ts) end -function expr_offset_to_maybe_token(ex::EXPR, offset::Integer, document::Document)::Union{Nothing,SemanticToken} - kind = semantic_token_kind(ex) +function maybe_get_token_from_expr_with_state(ex::EXPR, state::ExpressionVisitorState)::Union{Nothing,SemanticToken} + kind = semantic_token_kind(ex, state.external_env) if kind === nothing return nothing end @@ -73,7 +76,7 @@ function expr_offset_to_maybe_token(ex::EXPR, offset::Integer, document::Documen name_offset = -1 end end - line, char = get_position_from_offset(document, offset) + line, char = get_position_from_offset(state.document, state.offset) return SemanticToken( line, char, @@ -85,30 +88,39 @@ end mutable struct ExpressionVisitorState collected_tokens::Vector{SemanticToken} + # current offset per EXPR::fullspan (starts at 0) + offset::Integer + # access to positioning (used with offset) document::Document + # read-only + external_env::StaticLint.ExternalEnv end -ExpressionVisitorState(d::Document) = ExpressionVisitorState(SemanticToken[], d) +ExpressionVisitorState(args...) = ExpressionVisitorState(SemanticToken[], 0, args...) -function visit_every_expression_with_offset(expr_in::EXPR, state::ExpressionVisitorState, offset::Integer=0)::Nothing +""" + +Update state's offset with each-of expr_in's fullspan after visiting them +""" +function visit_every_expression(expr_in::EXPR, state::ExpressionVisitorState)::Nothing for e ∈ expr_in # ( maybe ) collect this expression - maybe_token = expr_offset_to_maybe_token(e, offset, state.document) + maybe_token = maybe_get_token_from_expr_with_state(e, state) if maybe_token !== nothing push!(state.collected_tokens, maybe_token) end # recurse into e's subtrees if !isempty(e) - visit_every_expression_with_offset(e, state, offset) + visit_every_expression(e, state) end - offset += e.fullspan + state.offset += e.fullspan end end -function every_semantic_token(document::Document) +function every_semantic_token(document::Document, external_env::StaticLint.ExternalEnv) root_expr = getcst(document) - state = ExpressionVisitorState(document) - visit_every_expression_with_offset(root_expr, state) + state = ExpressionVisitorState(document, external_env) + visit_every_expression(root_expr, state) state.collected_tokens end @@ -118,7 +130,8 @@ Get the semantic token kind for `expr`, which is assumed to be an identifier See CSTParser.jl/src/interface.jl """ -function semantic_token_kind(expr::EXPR)::Union{String,Nothing} +function semantic_token_kind(expr::EXPR, external_env::StaticLint.ExternalEnv)::Union{String,Nothing} + # TODO felipe use external_env return if C.isidentifier(expr) SemanticTokenKinds.Variable From 3661b94d0330eb38bccdb1eeaf1a70376a0ccaaa Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 2 Dec 2022 10:45:53 -0300 Subject: [PATCH 21/26] fix "struct not defined (yet)" --- src/requests/semantic.jl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index af934c32..45f89e3e 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -55,6 +55,17 @@ function textDocument_semanticTokens_full_request(params::SemanticTokensParams, return semantic_tokens(ts) end +mutable struct ExpressionVisitorState + collected_tokens::Vector{SemanticToken} + # current offset per EXPR::fullspan (starts at 0) + offset::Integer + # access to positioning (used with offset) + document::Document + # read-only + external_env::StaticLint.ExternalEnv +end +ExpressionVisitorState(args...) = ExpressionVisitorState(SemanticToken[], 0, args...) + function maybe_get_token_from_expr_with_state(ex::EXPR, state::ExpressionVisitorState)::Union{Nothing,SemanticToken} kind = semantic_token_kind(ex, state.external_env) if kind === nothing @@ -86,16 +97,6 @@ function maybe_get_token_from_expr_with_state(ex::EXPR, state::ExpressionVisitor ) end -mutable struct ExpressionVisitorState - collected_tokens::Vector{SemanticToken} - # current offset per EXPR::fullspan (starts at 0) - offset::Integer - # access to positioning (used with offset) - document::Document - # read-only - external_env::StaticLint.ExternalEnv -end -ExpressionVisitorState(args...) = ExpressionVisitorState(SemanticToken[], 0, args...) """ From a75ab25caf0a763581a08caf0dbc994b0bfbb6a9 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Fri, 2 Dec 2022 10:46:10 -0300 Subject: [PATCH 22/26] cleaner test output --- test/requests/test_semantic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requests/test_semantic.jl b/test/requests/test_semantic.jl index d24f8cd7..0e9fd42d 100644 --- a/test/requests/test_semantic.jl +++ b/test/requests/test_semantic.jl @@ -6,6 +6,6 @@ a=1 """) let _LS = LanguageServer - @test token_full_test() == _LS.SemanticTokens(UInt32[0, 0, 9, 5, _LS.semantic_token_encoding(_LS.SemanticTokenKinds.Variable)]) + @test Int64.(token_full_test().data) == Int64.(_LS.SemanticTokens(UInt32[0, 0, 9, 5, _LS.semantic_token_encoding(_LS.SemanticTokenKinds.Variable)]).data) end end From 110e98e3cf6c9156a7dccaf58683a36e14dbe460 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 14 Dec 2022 10:25:02 -0300 Subject: [PATCH 23/26] fix basic test --- test/requests/test_semantic.jl | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/requests/test_semantic.jl b/test/requests/test_semantic.jl index 0e9fd42d..9c7e3e3e 100644 --- a/test/requests/test_semantic.jl +++ b/test/requests/test_semantic.jl @@ -3,9 +3,20 @@ include("../test_shared_server.jl") doc = settestdoc(""" -a=1 +a=123 """) - let _LS = LanguageServer - @test Int64.(token_full_test().data) == Int64.(_LS.SemanticTokens(UInt32[0, 0, 9, 5, _LS.semantic_token_encoding(_LS.SemanticTokenKinds.Variable)]).data) + let _LS = LanguageServer, + _encoding=_LS.semantic_token_encoding + @test Int64.(token_full_test().data) == Int64.(_LS.SemanticTokens( + UInt32[0, 0, # first line, first column + 1, # «a» + _encoding(_LS.SemanticTokenKinds.Variable), 0, + 0, 1, + 1, # «=» + _encoding(_LS.SemanticTokenKinds.Operator), 0, + 0, 2, + 3, # «123» + _encoding(_LS.SemanticTokenKinds.Number), 0, + ]).data) end end From 2fad6639c5e7e6d90c6ce5fa2cff15ed5546322d Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 14 Dec 2022 14:11:50 -0300 Subject: [PATCH 24/26] explicitly use absolute-value pos, unify and sort tokens --- src/requests/semantic.jl | 110 +++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 45f89e3e..237cf655 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -1,11 +1,11 @@ const C = CSTParser -struct SemanticToken - # token line number, relative to the previous token - deltaLine::UInt32 - # token start character, relative to the previous token - # (relative to 0 or the previous token’s start if they are on the same line) - deltaStart::UInt32 +# Struct-like version of a semantic token before being flattened into 5-number-pair +struct NonFlattenedSemanticToken + # token line number + line::UInt32 + # token start character within line + start::UInt32 # the length of the token. length::UInt32 # will be looked up in SemanticTokensLegend.tokenTypes @@ -14,28 +14,21 @@ struct SemanticToken tokenModifiers::UInt32 end -function SemanticToken(deltaLine::UInt32, - deltaStart::UInt32, - length::UInt32, - tokenType::String, - tokenModifiers::String) - # FIXME look up int encodings for tokenType and tokenModifiers - SemanticToken( - deltaLine, - deltaStart, - length, - semantic_token_encoding(tokenType), - 0 # FIXME look up int encodings for tokenType and tokenModifiers - ) -end +""" + +Map collection of tokens into SemanticTokens + +Note: currently uses absolute position +""" function semantic_tokens(tokens)::SemanticTokens + # TODO implement relative position (track last token) token_data_size = length(tokens) * 5 token_data = Vector{UInt32}(undef, token_data_size) - for (i_token, token::SemanticToken) ∈ zip(1:5:token_data_size, tokens) + for (i_token, token::NonFlattenedSemanticToken) ∈ zip(1:5:token_data_size, tokens) token_data[i_token:i_token+4] = [ - token.deltaLine, - token.deltaStart, + token.line, + token.start, token.length, token.tokenType, token.tokenModifiers @@ -51,22 +44,36 @@ function textDocument_semanticTokens_full_request(params::SemanticTokensParams, external_env = getenv(d, server) - ts = collect(SemanticToken, every_semantic_token(d, external_env)) - return semantic_tokens(ts) + repeated_tokens = collect(NonFlattenedSemanticToken, every_semantic_token(d, external_env)) + sort!(repeated_tokens, lt=(l,r)->begin + (l.line, l.start) < (r.line, r.start) + end) + return semantic_tokens(unique(repeated_tokens)) end +# TODO visit expressions correctly and collect tokens into a Vector, rather than a Set (see visit_every_expression() ) +TokenCollection=Set{NonFlattenedSemanticToken} mutable struct ExpressionVisitorState - collected_tokens::Vector{SemanticToken} - # current offset per EXPR::fullspan (starts at 0) - offset::Integer - # access to positioning (used with offset) + collected_tokens::TokenCollection + # access to positioning (used with offset, see visit_every_expression() ) document::Document # read-only external_env::StaticLint.ExternalEnv end -ExpressionVisitorState(args...) = ExpressionVisitorState(SemanticToken[], 0, args...) +ExpressionVisitorState(args...) = ExpressionVisitorState(TokenCollection(), args...) + +""" +Adds token to state.collected only if maybe_get_token_from_expr() parsed an actual token +""" +function maybe_collect_token_from_expr(ex::EXPR, state::ExpressionVisitorState, offset::Integer) + maybe_token = maybe_get_token_from_expr(ex, state, offset) + if maybe_token !== nothing + push!(state.collected_tokens, maybe_token) + end + +end -function maybe_get_token_from_expr_with_state(ex::EXPR, state::ExpressionVisitorState)::Union{Nothing,SemanticToken} +function maybe_get_token_from_expr(ex::EXPR, state::ExpressionVisitorState, offset::Integer)::Union{Nothing,NonFlattenedSemanticToken} kind = semantic_token_kind(ex, state.external_env) if kind === nothing return nothing @@ -87,34 +94,33 @@ function maybe_get_token_from_expr_with_state(ex::EXPR, state::ExpressionVisitor name_offset = -1 end end - line, char = get_position_from_offset(state.document, state.offset) - return SemanticToken( - line, - char, - ex.span, - semantic_token_encoding(kind), - 0 - ) + line, char = get_position_from_offset(state.document, offset) + return NonFlattenedSemanticToken( + line, + char, + ex.span, + semantic_token_encoding(kind), + 0) end """ -Update state's offset with each-of expr_in's fullspan after visiting them +Visit each expression, collecting semantic-tokens into state + +Note: couldn't pack offset into ExpressionVisitorState and update, that's why it's a separate argument +TODO: not sure about how to recurse an EXPR and its sub-expressions. For now, that'll be covered by collecting them into a Set """ -function visit_every_expression(expr_in::EXPR, state::ExpressionVisitorState)::Nothing +function visit_every_expression(expr_in::EXPR, state::ExpressionVisitorState, offset=0)::Nothing + maybe_collect_token_from_expr(expr_in, state, offset) + + # recurse into this expression's expressions for e ∈ expr_in - # ( maybe ) collect this expression - maybe_token = maybe_get_token_from_expr_with_state(e, state) - if maybe_token !== nothing - push!(state.collected_tokens, maybe_token) - end + maybe_collect_token_from_expr(e, state, offset) - # recurse into e's subtrees - if !isempty(e) - visit_every_expression(e, state) - end - state.offset += e.fullspan + visit_every_expression(e, state, offset) + + offset += e.fullspan end end @@ -122,7 +128,7 @@ function every_semantic_token(document::Document, external_env::StaticLint.Exter root_expr = getcst(document) state = ExpressionVisitorState(document, external_env) visit_every_expression(root_expr, state) - state.collected_tokens + collect(state.collected_tokens) end From 0bc4a531260ca8e44fe012a6a40d264aef385d21 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 14 Dec 2022 14:34:02 -0300 Subject: [PATCH 25/26] add simple test this line looks off in my editor --- test/requests/test_semantic.jl | 42 ++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/test/requests/test_semantic.jl b/test/requests/test_semantic.jl index 9c7e3e3e..2fdc40fa 100644 --- a/test/requests/test_semantic.jl +++ b/test/requests/test_semantic.jl @@ -2,21 +2,35 @@ @testitem "simple token" begin include("../test_shared_server.jl") - doc = settestdoc(""" -a=123 - """) let _LS = LanguageServer, - _encoding=_LS.semantic_token_encoding + _encoding = _LS.semantic_token_encoding + + doc = settestdoc("""a=123""") + @test Int64.(token_full_test().data) == Int64.(_LS.SemanticTokens( + UInt32[0, 0, # first line, first column + 1, # «a» + _encoding(_LS.SemanticTokenKinds.Variable), 0, + 0, 1, + 1, # «=» + _encoding(_LS.SemanticTokenKinds.Operator), 0, + 0, 2, + 3, # «123» + _encoding(_LS.SemanticTokenKinds.Number), 0, + ]).data) + doc = settestdoc("""const C = CSTParser""") @test Int64.(token_full_test().data) == Int64.(_LS.SemanticTokens( - UInt32[0, 0, # first line, first column - 1, # «a» - _encoding(_LS.SemanticTokenKinds.Variable), 0, - 0, 1, - 1, # «=» - _encoding(_LS.SemanticTokenKinds.Operator), 0, - 0, 2, - 3, # «123» - _encoding(_LS.SemanticTokenKinds.Number), 0, - ]).data) + UInt32[0, 0, + 5, # «const » TODO WIP + _encoding(_LS.SemanticTokenKinds.Keyword), 0, + 0, 6, + 1, # «C» + _encoding(_LS.SemanticTokenKinds.Variable), 0, + 0, 8, + 1, # «=» + _encoding(_LS.SemanticTokenKinds.Operator), 0, + 0, 10, + 9, # «CSTParser» + _encoding(_LS.SemanticTokenKinds.Variable), 0, + ]).data) end end From 9a39129d2000e229ad8a90f4ee90f035f63cfe66 Mon Sep 17 00:00:00 2001 From: Felipe Lema Date: Wed, 14 Dec 2022 14:34:16 -0300 Subject: [PATCH 26/26] format --- src/requests/semantic.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl index 237cf655..80a85104 100644 --- a/src/requests/semantic.jl +++ b/src/requests/semantic.jl @@ -45,14 +45,14 @@ function textDocument_semanticTokens_full_request(params::SemanticTokensParams, external_env = getenv(d, server) repeated_tokens = collect(NonFlattenedSemanticToken, every_semantic_token(d, external_env)) - sort!(repeated_tokens, lt=(l,r)->begin - (l.line, l.start) < (r.line, r.start) - end) + sort!(repeated_tokens, lt=(l, r) -> begin + (l.line, l.start) < (r.line, r.start) + end) return semantic_tokens(unique(repeated_tokens)) end # TODO visit expressions correctly and collect tokens into a Vector, rather than a Set (see visit_every_expression() ) -TokenCollection=Set{NonFlattenedSemanticToken} +TokenCollection = Set{NonFlattenedSemanticToken} mutable struct ExpressionVisitorState collected_tokens::TokenCollection # access to positioning (used with offset, see visit_every_expression() ) @@ -96,11 +96,11 @@ function maybe_get_token_from_expr(ex::EXPR, state::ExpressionVisitorState, offs end line, char = get_position_from_offset(state.document, offset) return NonFlattenedSemanticToken( - line, - char, - ex.span, - semantic_token_encoding(kind), - 0) + line, + char, + ex.span, + semantic_token_encoding(kind), + 0) end