diff --git a/.gitignore b/.gitignore index abb9cc7..f7ae7ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/tmp/rust/ +/tmp/rust-sources.zip /target/ .not-ci .headless @@ -10,3 +12,5 @@ Vagrantfile /build /cluster-path +nb-configuration.xml +/nbproject/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..89628a3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "externals/rust-grammar"] + path = externals/rust-grammar + url = https://github.com/timboudreau/rust-grammar.git + diff --git a/externals/rust-grammar b/externals/rust-grammar new file mode 160000 index 0000000..b1bac7d --- /dev/null +++ b/externals/rust-grammar @@ -0,0 +1 @@ +Subproject commit b1bac7dd0995d626eff11aab5cd330e7e1228339 diff --git a/pom.xml b/pom.xml index 04a2a60..716b0da 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.github.drrb rust-netbeans - 1.0.0-SNAPSHOT + 1.0.3-SNAPSHOT nbm Rust NetBeans Plugin @@ -154,6 +154,12 @@ org-openide-util-ui ${netbeans.version} + + + org.antlr + antlr4-runtime + 4.7.1 + @@ -224,13 +230,12 @@ ${netbeans.version} - org.netbeans.api org-netbeans-modules-java-project @@ -320,6 +325,15 @@ guice 3.0 + + + org.netbeans.contrib.yenta + api + 1.1 + @@ -439,16 +453,22 @@ nbm-maven-plugin true + drrb.rust + warn GPL3 LICENSE.txt org.netbeans.modules:org-netbeans-modules-gsf-testrunner - impl + org.netbeans.modules.gsf.testrunner/2 > 1.0 org.netbeans.modules:org-netbeans-modules-gsf-testrunner-ui - impl + org.netbeans.modules.gsf.testrunner.ui > 1.0 + + + org.netbeans.modules:org-netbeans-modules-projectui + loose @@ -465,61 +485,146 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - true - -Xlint:unchecked - - - - - org.codehaus.mojo - javacc-maven-plugin - 2.6 + maven-antrun-plugin + 1.8 - generate-parser + copy-antlr-grammar generate-sources + + + + + + + + + + + + + + + + + + + + + + + + + + + + Downloading Rust sources to ${basedir}/tmp/rust-sources.zip if not present + + + + + + + + + + + + + - jjtree-javacc - jjdoc + run - - - net.java.dev.javacc - javacc - 7.0.2 - - - org.codehaus.mojo - exec-maven-plugin - 1.5.0 + org.antlr + antlr4-maven-plugin + 4.7.1 + + ${project.build.directory}/generated-sources/antlr-copied + + ${project.build.directory}/generated-sources/antlr-copied/imports + + true + - map-tokens - generate-sources + antlr - exec + antlr4 + generate-sources - ruby - - src/scripts/generate-rust-token-kind-enum - + true + true + + Java + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + true + -Xlint:unchecked + + @@ -568,13 +673,36 @@ maven-surefire-plugin + java.awt.headless=true: Stop tests hijacking UI on development machine + jna.nosys=true: Don't try to load JNA from the host system, use our dependency instead (needed for Appveyor, which has an old JNA on it) + noverify: Fix for JDK bug with PowerMock + --> -Djava.awt.headless=true -Djna.nosys=true -noverify - 1 + + + + false + 8 false + + + + + false + + + 100 + 100 + 100 + 100 + 100 + true + true + **/*IntegrationTest.java diff --git a/src/main/java/com/github/drrb/rust/netbeans/Installer.java b/src/main/java/com/github/drrb/rust/netbeans/Installer.java new file mode 100644 index 0000000..3ca06aa --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/Installer.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.netbeans.contrib.yenta.Yenta; + +/** + * Allows deployment in multiple versions of NetBeans, with the caveat + * that either of these modules may have broken its ABI in the meantime + * (but in practice, they haven't changed in years). + * + * @author Tim Boudreau + */ +public class Installer extends Yenta { + + @Override + protected Set siblings() { + return new HashSet<>(Arrays.asList("org.netbeans.modules.gsf.testrunner", "org.netbeans.modules.gsf.testrunner.ui")); + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java b/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java index 100ef8a..3a0ce00 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java +++ b/src/main/java/com/github/drrb/rust/netbeans/RustLanguage.java @@ -17,24 +17,27 @@ package com.github.drrb.rust.netbeans; import com.github.drrb.rust.netbeans.formatting.RustFormatter; -import com.github.drrb.rust.netbeans.highlighting.RustSemanticAnalyzer; import com.github.drrb.rust.netbeans.indexing.RustIndexSearcher; import com.github.drrb.rust.netbeans.indexing.RustIndexer; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParser; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrSemanticAnalyzer; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrStructureScanner; import org.netbeans.api.lexer.Language; -import org.netbeans.modules.csl.api.Formatter; -import org.netbeans.modules.csl.api.IndexSearcher; import org.netbeans.modules.csl.api.SemanticAnalyzer; import org.netbeans.modules.csl.spi.DefaultLanguageConfig; import org.netbeans.modules.csl.spi.LanguageRegistration; import org.netbeans.modules.parsing.spi.Parser; -import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; import org.netbeans.modules.parsing.spi.indexing.PathRecognizerRegistration; import org.openide.util.NbBundle; import java.util.Collections; import java.util.Set; +import org.netbeans.modules.csl.api.Formatter; +import org.netbeans.modules.csl.api.IndexSearcher; +import org.netbeans.modules.csl.api.StructureScanner; +import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; @LanguageRegistration(mimeType = RustLanguage.MIME_TYPE) @PathRecognizerRegistration(mimeTypes = RustLanguage.MIME_TYPE, sourcePathIds = RustLanguage.SOURCE_CLASSPATH_ID, libraryPathIds = RustLanguage.BOOT_CLASSPATH_ID, binaryLibraryPathIds = {}) @@ -70,13 +73,13 @@ public boolean isIdentifierChar(char c) { } @Override - public Language getLexerLanguage() { - return RustTokenId.language(); + public Language getLexerLanguage() { + return AntlrRustLanguageHierarchy.INSTANCE.language(); } @Override public Parser getParser() { - return new NetbeansRustParser(); + return new RustAntlrParser(); } @Override @@ -91,19 +94,28 @@ public Formatter getFormatter() { @Override public SemanticAnalyzer getSemanticAnalyzer() { - return new RustSemanticAnalyzer(); + return new RustAntlrSemanticAnalyzer(); + } + + @Override + public StructureScanner getStructureScanner() { + return new RustAntlrStructureScanner(); } @Override public EmbeddingIndexerFactory getIndexerFactory() { return new RustIndexer.Factory(); } - @Override public IndexSearcher getIndexSearcher() { return new RustIndexSearcher(); } + @Override + public boolean hasStructureScanner() { + return true; + } + //TODO: are these required? Is the annotation enough? @Override public Set getLibraryPathIds() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java b/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java index 2b46343..6044a9e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java +++ b/src/main/java/com/github/drrb/rust/netbeans/cargo/CargoConfig.java @@ -16,8 +16,10 @@ */ package com.github.drrb.rust.netbeans.cargo; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.forLiteralName; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBrace; import com.github.drrb.rust.netbeans.rustbridge.RustCrateType; import com.github.drrb.rust.netbeans.util.GsfUtilitiesHack; import com.github.drrb.rust.netbeans.util.Template; @@ -26,7 +28,6 @@ import org.netbeans.api.lexer.TokenSequence; import org.openide.filesystems.FileObject; import org.openide.filesystems.FileUtil; -import org.openide.text.NbDocument; import javax.swing.text.Document; import java.io.FileNotFoundException; @@ -69,11 +70,14 @@ private Iterable modFiles(FileObject sourceFile) { @Override public void run() { - TokenSequence rustTokens = new RustLexUtils().getRustTokenSequence(document, 0); + final AntlrTokenID mod = forLiteralName("mod"); + final AntlrTokenID lbrace = leftBrace(); + assert mod != null : "'mod' missing from vocabulary"; + TokenSequence rustTokens = new AntlrRustLexUtils().getRustTokenSequence(document, 0); lookingForModDeclarations: while (rustTokens.moveNext()) { - if (rustTokens.token().id() == RustTokenId.MOD && rustTokens.moveNext()) { - if (rustTokens.moveNext() && rustTokens.token().id() == RustTokenId.LEFT_BRACE) { + if (rustTokens.token().id() == mod && rustTokens.moveNext()) { + if (rustTokens.moveNext() && rustTokens.token().id() == lbrace) { // It's a mod literal continue lookingForModDeclarations; } else { @@ -117,7 +121,11 @@ public List getCrates() { String libCratePath = libCrate.getString("path"); if (libCratePath != null) { FileObject crateFile = baseDir.getFileObject(libCratePath); - List libCrateTypes = new LinkedList<>(libCrate.getList("crate-type")); + List found = libCrate.getList("crate-type"); + List libCrateTypes = new LinkedList<>(); + if (found != null) { + libCrateTypes.addAll(found); + } if (libCrateTypes.isEmpty()) { libCrateTypes.add("rlib"); } diff --git a/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java b/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java index 2d2e117..d804b1c 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java +++ b/src/main/java/com/github/drrb/rust/netbeans/commandrunner/CommandRunnerUi.java @@ -102,6 +102,8 @@ public void watch(Process process) { process.destroy(); } finally { progressHandle.finish(); + io.getOut().close(); + io.getErr().close(); } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java index f55ef85..5801ee9 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java +++ b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustDocumentFormatter.java @@ -16,8 +16,10 @@ */ package com.github.drrb.rust.netbeans.formatting; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenSequence; @@ -31,6 +33,7 @@ import javax.swing.text.Position; import java.util.LinkedList; import java.util.List; +import org.netbeans.api.lexer.Language; /** * @@ -38,11 +41,11 @@ public class RustDocumentFormatter { private final RustFormatter formatter; - private final NetbeansRustParserResult parseResult; + private final RustAntlrParserResult parseResult; private final BaseDocument document; private final Context context; - RustDocumentFormatter(RustFormatter formatter, NetbeansRustParserResult parseResult, BaseDocument document, Context context) { + RustDocumentFormatter(RustFormatter formatter, RustAntlrParserResult parseResult, BaseDocument document, Context context) { this.formatter = formatter; this.parseResult = parseResult; this.document = document; @@ -51,20 +54,24 @@ public class RustDocumentFormatter { public void format() { final Snapshot snapshot = parseResult.getSnapshot(); + final Language language = AntlrRustLanguageHierarchy.INSTANCE.language(); + final AntlrTokenID leftBrace = CommonRustTokenIDs.leftBrace(); + final AntlrTokenID rightBrace = CommonRustTokenIDs.rightBrace(); + final AntlrTokenID semicolon = CommonRustTokenIDs.semicolon(); try { List delimiters = new LinkedList<>(); TokenHierarchy tokenHierarchy = snapshot.getTokenHierarchy(); - TokenSequence tokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence tokenSequence = tokenHierarchy.tokenSequence(language); tokenSequence.move(0); while (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); + Token token = tokenSequence.token(); int tokenOffset = tokenSequence.offset(); - if (token.id() == RustTokenId.LEFT_BRACE) { + if (token.id() == leftBrace) { delimiters.add(new Delimiter(DelimiterType.OPEN_BRACE, tokenOffset)); - } else if (token.id() == RustTokenId.RIGHT_BRACE) { + } else if (token.id() == rightBrace) { delimiters.add(new Delimiter(DelimiterType.CLOSE_BRACE, tokenOffset)); - } else if (token.id() == RustTokenId.SEMICOLON) { + } else if (token.id() == semicolon) { delimiters.add(new Delimiter(DelimiterType.SEMICOLON, tokenOffset)); } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java index b0e89bc..c612a0e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java +++ b/src/main/java/com/github/drrb/rust/netbeans/formatting/RustFormatter.java @@ -16,8 +16,7 @@ */ package com.github.drrb.rust.netbeans.formatting; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; import static java.lang.Character.isWhitespace; import javax.swing.text.BadLocationException; import javax.swing.text.Document; @@ -35,7 +34,7 @@ public class RustFormatter implements Formatter { @Override public void reformat(Context context, ParserResult compilationInfo) { - NetbeansRustParserResult parseResult = (NetbeansRustParser.NetbeansRustParserResult) compilationInfo; + RustAntlrParserResult parseResult = (RustAntlrParserResult) compilationInfo; final BaseDocument document = (BaseDocument) context.document(); final RustDocumentFormatter formatter = new RustDocumentFormatter(this, parseResult, document, context); document.runAtomic(() -> { diff --git a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java index 1176fa9..bc75f98 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java +++ b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustBracesMatcher.java @@ -17,8 +17,14 @@ package com.github.drrb.rust.netbeans.highlighting; import com.github.drrb.rust.netbeans.RustLanguage; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftAngleBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBrace; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightAngleBracket; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightBrace; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightBracket; import org.netbeans.api.editor.mimelookup.MimeRegistration; import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenSequence; @@ -30,6 +36,9 @@ import javax.swing.text.BadLocationException; import java.util.logging.Level; import java.util.logging.Logger; +import org.netbeans.api.lexer.TokenId; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.leftParen; +import static com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs.rightParen; /** * @@ -43,7 +52,7 @@ public static class Factory implements BracesMatcherFactory { @Override public BracesMatcher createMatcher(MatcherContext context) { - return new RustBracesMatcher(context, new RustLexUtils()); + return new RustBracesMatcher(context, new AntlrRustLexUtils()); // TODO: is our implementation better than just doing this?: //return BracesMatcherSupport.defaultMatcher(context, -1, -1); // Probably, because it's dealing with tokens instead of characters @@ -51,9 +60,9 @@ public BracesMatcher createMatcher(MatcherContext context) { } } private final MatcherContext context; - private final RustLexUtils rustLexUtils; + private final AntlrRustLexUtils rustLexUtils; - public RustBracesMatcher(MatcherContext context, RustLexUtils rustLexUtils) { + public RustBracesMatcher(MatcherContext context, AntlrRustLexUtils rustLexUtils) { this.context = context; this.rustLexUtils = rustLexUtils; } @@ -64,7 +73,7 @@ public int[] findOrigin() throws InterruptedException, BadLocationException { document.readLock(); try { int offset = context.getSearchOffset(); - TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); + TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); if (tokenSequence == null) { LOGGER.warning("Couldn't get Rust token sequence for braces matching"); return null; @@ -76,12 +85,12 @@ public int[] findOrigin() throws InterruptedException, BadLocationException { } } - private OffsetRange getBraceAtOffset(TokenSequence tokenSequence, int offset) { + private OffsetRange getBraceAtOffset(TokenSequence tokenSequence, int offset) { tokenSequence.move(offset); if (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - for (BracePair bracePair : BracePair.values()) { - if (token.id() == bracePair.open || token.id() == bracePair.close) { + Token token = tokenSequence.token(); + for (BracePair bracePair : AntlrBracePair.values()) { + if (token.id() == bracePair.open() || token.id() == bracePair.close()) { return OffsetRange.ofCurrentToken(tokenSequence); } } @@ -97,7 +106,7 @@ public int[] findMatches() throws InterruptedException, BadLocationException { document.readLock(); try { int offset = context.getSearchOffset(); - TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); + TokenSequence tokenSequence = rustLexUtils.getRustTokenSequence(document, offset); if (tokenSequence == null) { LOGGER.warning("Couldn't get Rust token sequence for braces matching"); return null; @@ -109,14 +118,14 @@ public int[] findMatches() throws InterruptedException, BadLocationException { } } - private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence tokenSequence, int offset) { + private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence tokenSequence, int offset) { tokenSequence.move(offset); if (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - for (BracePair bracePair : BracePair.values()) { - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + for (BracePair bracePair : AntlrBracePair.values()) { + if (token.id() == bracePair.open()) { return findCloseBraceForward(tokenSequence, bracePair); - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { return findOpenBraceBackward(tokenSequence, bracePair); } } @@ -126,14 +135,14 @@ private OffsetRange getBraceMatchingTheOneAtOffset(TokenSequence to return OffsetRange.NONE; } - private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { + private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { int balance = 0; while (tokenSequence.moveNext()) { - Token token = tokenSequence.token(); - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + if (token.id() == bracePair.open()) { balance++; - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { if (balance == 0) { return OffsetRange.ofCurrentToken(tokenSequence); } @@ -144,17 +153,17 @@ private static OffsetRange findCloseBraceForward(TokenSequence tokenSequence, BracePair bracePair) { + private static OffsetRange findOpenBraceBackward(TokenSequence tokenSequence, BracePair bracePair) { int balance = 0; while (tokenSequence.movePrevious()) { - Token token = tokenSequence.token(); - if (token.id() == bracePair.open) { + Token token = tokenSequence.token(); + if (token.id() == bracePair.open()) { if (balance == 0) { return OffsetRange.ofCurrentToken(tokenSequence); } balance++; - } else if (token.id() == bracePair.close) { + } else if (token.id() == bracePair.close()) { balance--; } } @@ -162,19 +171,34 @@ private static OffsetRange findOpenBraceBackward(TokenSequence { +public class RustCompileErrorHighlighter extends ParserResultTask { private static final Logger LOG = Logger.getLogger(RustCompileErrorHighlighter.class.getName()); private static final RequestProcessor EXECUTOR = new RequestProcessor("Rust Compile", 12); -// @MimeRegistration(mimeType = RustLanguage.MIME_TYPE, service = TaskFactory.class) + @MimeRegistration(mimeType = RustLanguage.MIME_TYPE, service = TaskFactory.class) public static class Factory extends TaskFactory { @Override @@ -67,14 +69,7 @@ public Collection create(Snapshot snapshot) { } @Override - public void run(NetbeansRustParserResult parseResult, SchedulerEvent event) { - try { - if (parseResult.isFailure()) { - return; - } - } catch (ParseException ex) { - Exceptions.printStackTrace(ex); - } + public void run(RustAntlrParserResult parseResult, SchedulerEvent event) { final Snapshot snapshot = parseResult.getSnapshot(); EXECUTOR.post(new Runnable() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java deleted file mode 100644 index ac4755e..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/highlighting/RustSemanticAnalyzer.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.highlighting; - -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; -import com.github.drrb.rust.netbeans.parsing.javacc.*; -import org.netbeans.modules.csl.api.ColoringAttributes; -import org.netbeans.modules.csl.api.OffsetRange; -import org.netbeans.modules.csl.api.SemanticAnalyzer; -import org.netbeans.modules.parsing.spi.ParseException; -import org.netbeans.modules.parsing.spi.Scheduler; -import org.netbeans.modules.parsing.spi.SchedulerEvent; -import org.openide.util.Exceptions; - -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -import static com.github.drrb.rust.netbeans.parsing.javacc.ParseUtil.offsetRange; -import static org.netbeans.modules.csl.api.ColoringAttributes.*; - -/** - * - */ -public class RustSemanticAnalyzer extends SemanticAnalyzer { - private static final Logger LOG = Logger.getLogger(RustSemanticAnalyzer.class.getName()); - private final Map> highlights = new HashMap<>(); - private final AtomicBoolean cancelled = new AtomicBoolean(); - - @Override - public void run(NetbeansRustParserResult result, SchedulerEvent event) { - highlights.clear(); - cancelled.set(false); //TODO: respect this cancellation in the visitors - - try { - SimpleNode rootNode = result.rootNode(); - rootNode.jjtAccept(new FunctionNameHighlighter(), null); - rootNode.jjtAccept(new StructHighlighter(), null); - rootNode.jjtAccept(new AnnotationHighlighter(), null); - } catch (ParseException ex) { - Exceptions.printStackTrace(ex); - } - } - - @Override - public Map> getHighlights() { - return new HashMap<>(highlights); - } - - @Override - public int getPriority() { - return 0; - } - - @Override - public Class getSchedulerClass() { - return Scheduler.EDITOR_SENSITIVE_TASK_SCHEDULER; - } - - @Override - public void cancel() { - cancelled.set(true); - } - - private class FunctionNameHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTfunctionName functionNameNode, Object data) { - RustToken identifier = (RustToken) functionNameNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), EnumSet.of(STATIC, METHOD)); - return null; - } - } - - private class StructHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTstructName structNameNode, Object data) { - RustToken identifier = (RustToken) structNameNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), CLASS_SET); - return null; - } - - @Override - public Object visit(ASTstructField fieldNode, Object data) { - RustToken identifier = (RustToken) fieldNode.jjtGetFirstToken(); - highlights.put(identifier.offsetRange(), FIELD_SET); - return null; - } - } - - private class AnnotationHighlighter extends RustParserDefaultVisitor { - @Override - public Object visit(ASTAnnotation node, Object data) { - highlights.put(offsetRange(node), EnumSet.of(ANNOTATION_TYPE)); - return null; - } - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java b/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java index 6c5f35e..7becefe 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java +++ b/src/main/java/com/github/drrb/rust/netbeans/indexing/RustIndexer.java @@ -16,28 +16,24 @@ */ package com.github.drrb.rust.netbeans.indexing; -import com.github.drrb.rust.netbeans.parsing.NetbeansRustParser.NetbeansRustParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; +import com.github.drrb.rust.netbeans.parsing.antlr.RustStructureItem; import com.github.drrb.rust.netbeans.parsing.index.RustStruct; import com.github.drrb.rust.netbeans.parsing.index.RustStructBody; -import com.github.drrb.rust.netbeans.parsing.javacc.ASTStructItem; -import com.github.drrb.rust.netbeans.parsing.javacc.ASTstructName; -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserDefaultVisitor; +import com.github.drrb.rust.netbeans.parsing.index.RustStructField; import org.netbeans.modules.parsing.api.Snapshot; -import org.netbeans.modules.parsing.spi.ParseException; import org.netbeans.modules.parsing.spi.Parser; import org.netbeans.modules.parsing.spi.indexing.Context; import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexer; import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory; import org.netbeans.modules.parsing.spi.indexing.Indexable; -import org.openide.util.Exceptions; import java.io.IOException; -import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import static com.github.drrb.rust.netbeans.parsing.javacc.ParseUtil.offsetRange; +import java.util.ArrayList; /** * @@ -58,14 +54,47 @@ protected void index(Indexable file, Parser.Result parserResult, Context context LOGGER.log(Level.INFO, "RustIndexer.index({0})", file.getRelativePath()); try { RustIndexWriter indexWriter = index.createIndexWriter(context); - NetbeansRustParserResult parseResult = (NetbeansRustParserResult) parserResult; - IndexingRustVisitor visitor = new IndexingRustVisitor(); - parseResult.rootNode().jjtAccept(visitor, null); - for (RustStruct struct : visitor.structs) { + List structs = findIndexable((RustAntlrParserResult) parserResult); + for (RustStruct struct : structs) { + LOGGER.log(Level.INFO, " include ({0})", struct); indexWriter.write(file, struct); } - } catch (IOException | ParseException ex) { - Exceptions.printStackTrace(ex); + } catch (IOException ex) { + LOGGER.log(Level.WARNING, "Exception indexing " + file.getRelativePath(), ex); + } + } + + private List findIndexable(RustAntlrParserResult parserResult) { + List result = new ArrayList<>(); + for (RustStructureItem structureItem : parserResult.structureItems()) { + RustStruct struct = toRustStruct(structureItem); + if (struct != null) { + result.add(struct); + } + } + return result; + } + + private RustStruct toRustStruct(RustStructureItem item) { + switch (item.rustKind()) { + case ATTR: + case LIFETIME: + case TYPE_REFERENCE: + case TYPE: + return null; + case STRUCT: + RustStruct.Builder structBuilder = RustStruct.builder() + .setOffsetRange(item.range()).setName(item.getName()); + RustStructBody.Builder structBodyBuilder = RustStructBody.builder(); + for (RustStructureItem child : item.getNestedItems()) { + structBodyBuilder.addField(new RustStructField(child.getName(), child.range())); + } + structBuilder.setBody(structBodyBuilder.build()); + return structBuilder.build(); +// case ENUM : +// RustEnum.Builder enumBuilder = RustEnum.builder(); + default: + return null; } } @@ -98,27 +127,4 @@ public int getIndexVersion() { return VERSION; } } - - private static class IndexingRustVisitor extends RustParserDefaultVisitor { - private final List structs = new LinkedList<>(); - - @Override - public Object visit(ASTStructItem structNode, Object data) { - RustStruct.Builder struct = RustStruct.builder() - .setOffsetRange(offsetRange(structNode)); - RustStructBody.Builder structBody = RustStructBody.builder(); - structNode.jjtAccept(new RustParserDefaultVisitor() { - - @Override - public Object visit(ASTstructName node, Object data) { - struct.setName(node.jjtGetFirstToken().image); - return null; - } - - }, null); - struct.setBody(structBody.build()); - structs.add(struct.build()); - return null; - } - } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java b/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java index 5a4da79..761448e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java +++ b/src/main/java/com/github/drrb/rust/netbeans/keypress/RustBreakInterceptor.java @@ -17,8 +17,6 @@ package com.github.drrb.rust.netbeans.keypress; import com.github.drrb.rust.netbeans.RustLanguage; -import com.github.drrb.rust.netbeans.parsing.RustLexUtils; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; import org.netbeans.api.editor.mimelookup.MimePath; import org.netbeans.api.editor.mimelookup.MimeRegistration; import org.netbeans.api.lexer.TokenId; @@ -30,7 +28,9 @@ import javax.swing.text.BadLocationException; import java.util.concurrent.atomic.AtomicBoolean; -import static com.github.drrb.rust.netbeans.parsing.RustTokenId.*; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLexUtils; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; /** * @@ -48,14 +48,16 @@ public boolean beforeInsert(Context context) throws BadLocationException { @Override public void insert(MutableContext ctx) throws BadLocationException { if (cancelled.get()) return; + final AntlrTokenID leftBrace = CommonRustTokenIDs.leftBrace(); + final AntlrTokenID rightBrace = CommonRustTokenIDs.rightBrace(); ContextHolder context = new ContextHolder(ctx); - if (context.previousTokenKind() != LEFT_BRACE) { + if (context.previousTokenKind() != leftBrace) { return; // Only insert a close brace after an open brace } else if (context.nextRowIndent() > context.currentRowIndent()) { return; // There's already stuff in this block - } else if (context.nextTokenKind() == RIGHT_BRACE && context.currentRowIndent() == context.nextRowIndent()) { + } else if (context.nextTokenKind() == rightBrace && context.currentRowIndent() == context.nextRowIndent()) { return; // There's already a closing brace } @@ -74,21 +76,21 @@ public void cancelled(Context context) { private static class ContextHolder { private final MutableContext context; - private final TokenSequence tokenSequence; + private final TokenSequence tokenSequence; private Integer currentRowIndent; private Integer nextRowIndent; private ContextHolder(MutableContext context) { this.context = context; - this.tokenSequence = new RustLexUtils().getRustTokenSequence(context.getDocument(), context.getCaretOffset()); + this.tokenSequence = new AntlrRustLexUtils().getRustTokenSequence(context.getDocument(), context.getCaretOffset()); tokenSequence.move(context.getCaretOffset()); } - private RustTokenId previousTokenKind() { + private TokenId previousTokenKind() { return findNonWhitespaceToken(Direction.BACKWARD); } - private RustTokenId nextTokenKind() { + private TokenId nextTokenKind() { return findNonWhitespaceToken(Direction.FORWARD); } @@ -113,10 +115,10 @@ private int nextRowIndent() throws BadLocationException { return nextRowIndent; } - private RustTokenId findNonWhitespaceToken(Direction direction) { + private TokenId findNonWhitespaceToken(Direction direction) { while (direction.move(tokenSequence)) { - RustTokenId nextTokenKind = tokenSequence.token().id(); - if (nextTokenKind != WHITESPACE) { + TokenId nextTokenKind = tokenSequence.token().id(); + if (nextTokenKind != CommonRustTokenIDs.forSymbolicName("Whitespace")) { return nextTokenKind; } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java deleted file mode 100644 index 62ab141..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustLexer.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.JavaccCharStream; -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserTokenManager; -import com.github.drrb.rust.netbeans.parsing.javacc.TokenMgrError; -import com.github.drrb.rust.netbeans.rustbridge.RustLexer; -import org.netbeans.api.lexer.Token; -import org.netbeans.spi.lexer.Lexer; -import org.netbeans.spi.lexer.LexerInput; -import org.netbeans.spi.lexer.LexerRestartInfo; - -import java.util.Stack; -import java.util.stream.IntStream; - -public class NetbeansRustLexer implements Lexer { - - private static final Token EOF_TOKEN = null; - - private final LexerRestartInfo info; - private final RustParserTokenManager tokenManager; - private final Stack unreturnedTokens = new Stack<>(); - private final LexerInput lexerInput; - private RustLexer lexer = RustLexer.NULL_LEXER; - - public NetbeansRustLexer(LexerRestartInfo info) { - this.info = info; - this.lexerInput = info.input(); - this.tokenManager = new RustParserTokenManager(new JavaccCharStream(info.input())); - } - - @Override - public Token nextToken() { - try { - com.github.drrb.rust.netbeans.parsing.javacc.RustToken token; - if (unreturnedTokens.empty()) { - token = (com.github.drrb.rust.netbeans.parsing.javacc.RustToken) tokenManager.getNextToken(); - log("parsed token: %s%n", token); - while (token.hasSpecialToken()) { - if (token.isEof()) { - info.input().backup(1); - } - log(" token %s has special token %s. pushing %s%n", token, token.specialToken(), token); - unreturnedTokens.push(token); - log(" backing up %s%n", token.image.length()); - lexerInput.backup(token.image.length()); - token = token.specialToken(); - } - } else { - token = unreturnedTokens.pop(); - log("using unreturned token %s%n", token); - IntStream.range(0, token.image.length()).forEach(x -> lexerInput.read()); - } - - log("token = %s%n", token); - log(" read = %s%n", lexerInput.readLength()); - if (token.isEof()) { - return EOF_TOKEN; - } else { - return info.tokenFactory().createToken(RustTokenId.get(token.kind)); - } - } catch (TokenMgrError e) { - return info.tokenFactory().createToken(RustTokenId.GARBAGE); - } - } - - private void log(String format, Object... args) { - //System.out.format(format, args); - } - - private void ensureLexerCreated() { - if (lexer == RustLexer.NULL_LEXER) { - String source = readWholeSource(); - backUp(charsReadThisToken()); - lexer = RustLexer.forString(source); - } - } - - private String readWholeSource() { - reading: - while (readOneCharacter() != LexerInput.EOF) { - continue reading; - } - return charactersReadSoFar(); - } - - protected void backUp(int length) { - info.input().backup(length); - } - - protected int charsReadThisToken() { - return info.input().readLengthEOF(); - } - - protected String charactersReadSoFar() { - return info.input().readText().toString(); - } - - protected int readOneCharacter() { - return info.input().read(); - } - - protected Token createToken(RustTokenId tokenType) { - return info.tokenFactory().createToken(tokenType); - } - - @Override - public Object state() { - return null; - } - - @Override - public void release() { - //lexer.release(); - } - -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java deleted file mode 100644 index 2245628..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/NetbeansRustParser.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.RustParser; -import com.github.drrb.rust.netbeans.parsing.javacc.RustToken; -import com.github.drrb.rust.netbeans.parsing.javacc.SimpleNode; -import org.netbeans.modules.csl.api.Error; -import org.netbeans.modules.csl.api.Severity; -import org.netbeans.modules.csl.spi.DefaultError; -import org.netbeans.modules.csl.spi.ParserResult; -import org.netbeans.modules.parsing.api.Snapshot; -import org.netbeans.modules.parsing.api.Task; -import org.netbeans.modules.parsing.spi.ParseException; -import org.netbeans.modules.parsing.spi.Parser; -import org.netbeans.modules.parsing.spi.SourceModificationEvent; -import org.openide.filesystems.FileObject; - -import javax.swing.event.ChangeListener; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -import static java.util.stream.Collectors.toList; - -/** - * - */ -public class NetbeansRustParser extends Parser { - private static final Logger LOG = Logger.getLogger(NetbeansRustParser.class.getName()); - private Snapshot snapshot; - private NetbeansRustParserResult result; - - @Override - public void parse(final Snapshot snapshot, Task task, SourceModificationEvent event) { - this.snapshot = snapshot; - String source = snapshot.getText().toString(); - - RustParser.Result parseResult = RustParser.parse(source); - result = NetbeansRustParserResult.complete(snapshot, parseResult); - } - - @Override - public NetbeansRustParserResult getResult(Task task) throws ParseException { - return result; - } - - @Override - public void addChangeListener(ChangeListener changeListener) { - } - - @Override - public void removeChangeListener(ChangeListener changeListener) { - } - - public static class NetbeansRustParserResult extends ParserResult { - - private final AtomicBoolean valid = new AtomicBoolean(true); - private final List diagnostics; - private final RustParser.Result parseResult; - - public NetbeansRustParserResult(Snapshot snapshot, RustParser.Result parseResult, List diagnostics) { - super(snapshot); - this.parseResult = parseResult; - this.diagnostics = Collections.unmodifiableList(diagnostics); - } - - public SimpleNode rootNode() throws ParseException { - //TODO: i think i've seen the valid field on the parser itself in an example. Where should it be? - //TODO: also, this seems to be invalidated before the first use. Why? -// if (!valid.get()) { -// throw new ParseException(); -// } - return parseResult.rootNode(); - } - - @Override - protected void invalidate() { - valid.set(false); - } - - @Override - public List getDiagnostics() { - return diagnostics; - } - - public static NetbeansRustParserResult complete(Snapshot snapshot, RustParser.Result parseResult) { - return new NetbeansRustParserResult(snapshot, parseResult, parseResult.syntaxErrors().stream().map(ex -> toError(snapshot, ex)).collect(toList())); - } - - private static DefaultError toError(Snapshot snapshot, com.github.drrb.rust.netbeans.parsing.javacc.ParseException e) { - RustToken currentToken = (RustToken) e.currentToken.next; - FileObject file = snapshot.getSource().getFileObject(); - return new DefaultError( - "rust.parse.message", - "Parse error", - e.getMessage(), - file, - currentToken.absoluteBeginPosition - 1, - currentToken.absoluteEndPosition - 1, - false, - Severity.ERROR - ); - } - - public boolean isFailure() throws ParseException { - return rootNode() != null; - } - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java index b147b47..62aff06 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/OffsetRustToken.java @@ -18,6 +18,7 @@ import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenHierarchy; +import org.netbeans.api.lexer.TokenId; import org.netbeans.api.lexer.TokenSequence; import org.netbeans.modules.csl.api.OffsetRange; @@ -26,16 +27,16 @@ */ public class OffsetRustToken { - public static OffsetRustToken atCurrentLocation(TokenSequence tokenSequence) { + public static OffsetRustToken atCurrentLocation(TokenSequence tokenSequence) { return new OffsetRustToken(tokenSequence.offsetToken()); } - private final Token token; + private final Token token; - private OffsetRustToken(Token offsetToken) { + private OffsetRustToken(Token offsetToken) { this.token = offsetToken; } - public RustTokenId id() { + public TokenId id() { return token.id(); } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java deleted file mode 100644 index 61b326d..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLanguageHierarchy.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.RustLanguage; -import org.netbeans.spi.lexer.LanguageHierarchy; -import org.netbeans.spi.lexer.Lexer; -import org.netbeans.spi.lexer.LexerRestartInfo; - -import java.util.Collection; -import java.util.EnumSet; - -import static java.util.Collections.unmodifiableSet; - -public class RustLanguageHierarchy extends LanguageHierarchy { - private static final Collection TOKEN_IDS = unmodifiableSet(EnumSet.allOf(RustTokenId.class)); - - @Override - protected Collection createTokenIds() { - return TOKEN_IDS; - } - - @Override - protected Lexer createLexer(LexerRestartInfo info) { - return new NetbeansRustLexer(info); - } - - @Override - protected String mimeType() { - return RustLanguage.MIME_TYPE; - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java deleted file mode 100644 index 973ca70..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustTokenId.java +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing; - -import com.github.drrb.rust.netbeans.parsing.javacc.RustParserConstants; -import org.netbeans.api.lexer.Language; -import org.netbeans.api.lexer.TokenId; - -public enum RustTokenId implements TokenId { - - EOF(RustParserConstants.EOF, TokenCategory.WHITESPACE), - WHITESPACE(RustParserConstants.WHITESPACE, TokenCategory.WHITESPACE), - DOC_COMMENT(RustParserConstants.DOC_COMMENT, TokenCategory.COMMENT), - INNER_DOC_COMMENT(RustParserConstants.INNER_DOC_COMMENT, TokenCategory.COMMENT), - LINE_COMMENT(RustParserConstants.LINE_COMMENT, TokenCategory.COMMENT), - BLOCK_COMMENT(RustParserConstants.BLOCK_COMMENT, TokenCategory.COMMENT), - DOC_BLOCK_COMMENT(RustParserConstants.DOC_BLOCK_COMMENT, TokenCategory.COMMENT), - INNER_DOC_BLOCK_COMMENT(RustParserConstants.INNER_DOC_BLOCK_COMMENT, TokenCategory.COMMENT), - NON_NULL(RustParserConstants.NON_NULL, TokenCategory.IDENTIFIER), - NON_SINGLE_QUOTE(RustParserConstants.NON_SINGLE_QUOTE, TokenCategory.IDENTIFIER), - NON_DOUBLE_QUOTE(RustParserConstants.NON_DOUBLE_QUOTE, TokenCategory.IDENTIFIER), - NON_EOL(RustParserConstants.NON_EOL, TokenCategory.IDENTIFIER), - ASCII(RustParserConstants.ASCII, TokenCategory.IDENTIFIER), - ASCII_NON_EOL(RustParserConstants.ASCII_NON_EOL, TokenCategory.IDENTIFIER), - ASCII_NON_SINGLE_QUOTE(RustParserConstants.ASCII_NON_SINGLE_QUOTE, TokenCategory.IDENTIFIER), - ASCII_NON_DOUBLE_QUOTE(RustParserConstants.ASCII_NON_DOUBLE_QUOTE, TokenCategory.IDENTIFIER), - STRING_LITERAL(RustParserConstants.STRING_LITERAL, TokenCategory.STRING), - RAW_STRING_LITERAL(RustParserConstants.RAW_STRING_LITERAL, TokenCategory.STRING), - CHAR_LITERAL(RustParserConstants.CHAR_LITERAL, TokenCategory.CHARACTER), - NUMBER_LITERAL(RustParserConstants.NUMBER_LITERAL, TokenCategory.NUMBER), - BYTE_LITERAL(RustParserConstants.BYTE_LITERAL, TokenCategory.CHARACTER), - BYTE_STRING_LITERAL(RustParserConstants.BYTE_STRING_LITERAL, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL(RustParserConstants.RAW_BYTE_STRING_LITERAL, TokenCategory.STRING), - STRING_BODY(RustParserConstants.STRING_BODY, TokenCategory.STRING), - CHAR_BODY(RustParserConstants.CHAR_BODY, TokenCategory.CHARACTER), - BYTE_BODY(RustParserConstants.BYTE_BODY, TokenCategory.IDENTIFIER), - COMMON_ESCAPE(RustParserConstants.COMMON_ESCAPE, TokenCategory.IDENTIFIER), - UNICODE_ESCAPE(RustParserConstants.UNICODE_ESCAPE, TokenCategory.IDENTIFIER), - FLOAT_SUFFIX(RustParserConstants.FLOAT_SUFFIX, TokenCategory.NUMBER), - EXPONENT(RustParserConstants.EXPONENT, TokenCategory.NUMBER), - DEC_LIT(RustParserConstants.DEC_LIT, TokenCategory.NUMBER), - HEX_DIGIT(RustParserConstants.HEX_DIGIT, TokenCategory.NUMBER), - OCT_DIGIT(RustParserConstants.OCT_DIGIT, TokenCategory.NUMBER), - DEC_DIGIT(RustParserConstants.DEC_DIGIT, TokenCategory.NUMBER), - NONZERO_DEC(RustParserConstants.NONZERO_DEC, TokenCategory.NUMBER), - RAW_STRING_LITERAL_3(RustParserConstants.RAW_STRING_LITERAL_3, TokenCategory.STRING), - RAW_STRING_LITERAL_2(RustParserConstants.RAW_STRING_LITERAL_2, TokenCategory.STRING), - RAW_STRING_LITERAL_1(RustParserConstants.RAW_STRING_LITERAL_1, TokenCategory.STRING), - RAW_STRING_LITERAL_0(RustParserConstants.RAW_STRING_LITERAL_0, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_3(RustParserConstants.RAW_BYTE_STRING_LITERAL_3, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_2(RustParserConstants.RAW_BYTE_STRING_LITERAL_2, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_1(RustParserConstants.RAW_BYTE_STRING_LITERAL_1, TokenCategory.STRING), - RAW_BYTE_STRING_LITERAL_0(RustParserConstants.RAW_BYTE_STRING_LITERAL_0, TokenCategory.STRING), - DOUBLE_COLON(RustParserConstants.DOUBLE_COLON, TokenCategory.SEPARATOR), - ARROW(RustParserConstants.ARROW, TokenCategory.SEPARATOR), - DOUBLE_ARROW(RustParserConstants.DOUBLE_ARROW, TokenCategory.OPERATOR), - HASH(RustParserConstants.HASH, TokenCategory.SEPARATOR), - LEFT_BRACKET(RustParserConstants.LEFT_BRACKET, TokenCategory.SEPARATOR), - RIGHT_BRACKET(RustParserConstants.RIGHT_BRACKET, TokenCategory.SEPARATOR), - LEFT_PAREN(RustParserConstants.LEFT_PAREN, TokenCategory.SEPARATOR), - RIGHT_PAREN(RustParserConstants.RIGHT_PAREN, TokenCategory.SEPARATOR), - LEFT_BRACE(RustParserConstants.LEFT_BRACE, TokenCategory.SEPARATOR), - RIGHT_BRACE(RustParserConstants.RIGHT_BRACE, TokenCategory.SEPARATOR), - COMMA(RustParserConstants.COMMA, TokenCategory.SEPARATOR), - COLON(RustParserConstants.COLON, TokenCategory.SEPARATOR), - PLUS(RustParserConstants.PLUS, TokenCategory.OPERATOR), - DASH(RustParserConstants.DASH, TokenCategory.OPERATOR), - STAR(RustParserConstants.STAR, TokenCategory.OPERATOR), - FORWARD_SLASH(RustParserConstants.FORWARD_SLASH, TokenCategory.OPERATOR), - PERCENT(RustParserConstants.PERCENT, TokenCategory.OPERATOR), - AMPERSAND(RustParserConstants.AMPERSAND, TokenCategory.OPERATOR), - PIPE(RustParserConstants.PIPE, TokenCategory.OPERATOR), - HAT(RustParserConstants.HAT, TokenCategory.OPERATOR), - DOUBLE_AMPERSAND(RustParserConstants.DOUBLE_AMPERSAND, TokenCategory.OPERATOR), - DOUBLE_PIPE(RustParserConstants.DOUBLE_PIPE, TokenCategory.OPERATOR), - LEFT_ANGLE_BRACKET(RustParserConstants.LEFT_ANGLE_BRACKET, TokenCategory.SEPARATOR), - RIGHT_ANGLE_BRACKET(RustParserConstants.RIGHT_ANGLE_BRACKET, TokenCategory.SEPARATOR), - SHIFT_LEFT(RustParserConstants.SHIFT_LEFT, TokenCategory.OPERATOR), - SHIFT_RIGHT(RustParserConstants.SHIFT_RIGHT, TokenCategory.OPERATOR), - LESS_THAN_EQUAL(RustParserConstants.LESS_THAN_EQUAL, TokenCategory.OPERATOR), - GREATER_THAN_EQUAL(RustParserConstants.GREATER_THAN_EQUAL, TokenCategory.OPERATOR), - SEMICOLON(RustParserConstants.SEMICOLON, TokenCategory.SEPARATOR), - DOUBLE_EQUALS(RustParserConstants.DOUBLE_EQUALS, TokenCategory.OPERATOR), - NOT_EQUAL(RustParserConstants.NOT_EQUAL, TokenCategory.OPERATOR), - PLUS_EQUALS(RustParserConstants.PLUS_EQUALS, TokenCategory.OPERATOR), - MINUS_EQUALS(RustParserConstants.MINUS_EQUALS, TokenCategory.OPERATOR), - TIMES_EQUALS(RustParserConstants.TIMES_EQUALS, TokenCategory.OPERATOR), - DIVIDE_EQUALS(RustParserConstants.DIVIDE_EQUALS, TokenCategory.OPERATOR), - MOD_EQUALS(RustParserConstants.MOD_EQUALS, TokenCategory.OPERATOR), - AND_EQUALS(RustParserConstants.AND_EQUALS, TokenCategory.OPERATOR), - OR_EQUALS(RustParserConstants.OR_EQUALS, TokenCategory.OPERATOR), - XOR_EQUALS(RustParserConstants.XOR_EQUALS, TokenCategory.OPERATOR), - SHIFT_LEFT_EQUALS(RustParserConstants.SHIFT_LEFT_EQUALS, TokenCategory.OPERATOR), - SHIFT_RIGHT_EQUALS(RustParserConstants.SHIFT_RIGHT_EQUALS, TokenCategory.OPERATOR), - BANG(RustParserConstants.BANG, TokenCategory.OPERATOR), - EQUALS(RustParserConstants.EQUALS, TokenCategory.OPERATOR), - DOT(RustParserConstants.DOT, TokenCategory.SEPARATOR), - DOUBLE_DOT(RustParserConstants.DOUBLE_DOT, TokenCategory.IDENTIFIER), - DOLLAR(RustParserConstants.DOLLAR, TokenCategory.SEPARATOR), - HASH_ROCKET(RustParserConstants.HASH_ROCKET, TokenCategory.SEPARATOR), - ABSTRACT(RustParserConstants.ABSTRACT, TokenCategory.IDENTIFIER), - ALIGNOF(RustParserConstants.ALIGNOF, TokenCategory.IDENTIFIER), - AS(RustParserConstants.AS, TokenCategory.KEYWORD), - BECOME(RustParserConstants.BECOME, TokenCategory.IDENTIFIER), - BOX(RustParserConstants.BOX, TokenCategory.IDENTIFIER), - BREAK(RustParserConstants.BREAK, TokenCategory.KEYWORD), - CONST(RustParserConstants.CONST, TokenCategory.KEYWORD), - CONTINUE(RustParserConstants.CONTINUE, TokenCategory.KEYWORD), - CRATE(RustParserConstants.CRATE, TokenCategory.KEYWORD), - DO(RustParserConstants.DO, TokenCategory.KEYWORD), - ELSE(RustParserConstants.ELSE, TokenCategory.KEYWORD), - ENUM(RustParserConstants.ENUM, TokenCategory.KEYWORD), - EXTERN(RustParserConstants.EXTERN, TokenCategory.KEYWORD), - FALSE(RustParserConstants.FALSE, TokenCategory.KEYWORD), - FINAL(RustParserConstants.FINAL, TokenCategory.KEYWORD), - FN(RustParserConstants.FN, TokenCategory.KEYWORD), - FOR(RustParserConstants.FOR, TokenCategory.KEYWORD), - IF(RustParserConstants.IF, TokenCategory.KEYWORD), - IMPL(RustParserConstants.IMPL, TokenCategory.KEYWORD), - IN(RustParserConstants.IN, TokenCategory.KEYWORD), - LET(RustParserConstants.LET, TokenCategory.KEYWORD), - LOOP(RustParserConstants.LOOP, TokenCategory.KEYWORD), - MACRO(RustParserConstants.MACRO, TokenCategory.KEYWORD), - MACRO_RULES(RustParserConstants.MACRO_RULES, TokenCategory.IDENTIFIER), - MATCH(RustParserConstants.MATCH, TokenCategory.KEYWORD), - MOD(RustParserConstants.MOD, TokenCategory.IDENTIFIER), - MOVE(RustParserConstants.MOVE, TokenCategory.IDENTIFIER), - MUT(RustParserConstants.MUT, TokenCategory.KEYWORD), - OFFSETOF(RustParserConstants.OFFSETOF, TokenCategory.IDENTIFIER), - OVERRIDE(RustParserConstants.OVERRIDE, TokenCategory.IDENTIFIER), - PRIV(RustParserConstants.PRIV, TokenCategory.KEYWORD), - PROC(RustParserConstants.PROC, TokenCategory.IDENTIFIER), - PUB(RustParserConstants.PUB, TokenCategory.KEYWORD), - PURE(RustParserConstants.PURE, TokenCategory.IDENTIFIER), - REF(RustParserConstants.REF, TokenCategory.IDENTIFIER), - RETURN(RustParserConstants.RETURN, TokenCategory.KEYWORD), - BIG_SELF(RustParserConstants.BIG_SELF, TokenCategory.IDENTIFIER), - SELF(RustParserConstants.SELF, TokenCategory.KEYWORD), - SIZEOF(RustParserConstants.SIZEOF, TokenCategory.KEYWORD), - STATIC(RustParserConstants.STATIC, TokenCategory.KEYWORD), - STRUCT(RustParserConstants.STRUCT, TokenCategory.KEYWORD), - SUPER(RustParserConstants.SUPER, TokenCategory.IDENTIFIER), - TRAIT(RustParserConstants.TRAIT, TokenCategory.KEYWORD), - TRUE(RustParserConstants.TRUE, TokenCategory.KEYWORD), - TYPE(RustParserConstants.TYPE, TokenCategory.KEYWORD), - TYPEOF(RustParserConstants.TYPEOF, TokenCategory.KEYWORD), - UNSAFE(RustParserConstants.UNSAFE, TokenCategory.KEYWORD), - UNSIZED(RustParserConstants.UNSIZED, TokenCategory.KEYWORD), - USE(RustParserConstants.USE, TokenCategory.KEYWORD), - VIRTUAL(RustParserConstants.VIRTUAL, TokenCategory.KEYWORD), - WHERE(RustParserConstants.WHERE, TokenCategory.KEYWORD), - WHILE(RustParserConstants.WHILE, TokenCategory.KEYWORD), - YIELD(RustParserConstants.YIELD, TokenCategory.KEYWORD), - IDENTIFIER(RustParserConstants.IDENTIFIER, TokenCategory.IDENTIFIER), - LABEL(RustParserConstants.LABEL, TokenCategory.IDENTIFIER), - XID_start(RustParserConstants.XID_start, TokenCategory.IDENTIFIER), - XID_continue(RustParserConstants.XID_continue, TokenCategory.IDENTIFIER), - GARBAGE(RustParserConstants.GARBAGE, TokenCategory.IDENTIFIER); - - public static final RustLanguageHierarchy LANGUAGE_HIERARCHY = new RustLanguageHierarchy(); - private static final RustTokenId[] LOOKUP; - static { - int highestValue = 0; - for (RustTokenId kind : values()) { - highestValue = highestValue > kind.javaccKind ? highestValue : kind.javaccKind; - } - LOOKUP = new RustTokenId[highestValue + 1]; - for (RustTokenId kind : values()) { - LOOKUP[kind.javaccKind] = kind; - } - } - - public static Language language() { - return LANGUAGE_HIERARCHY.language(); - } - - private final int javaccKind; - private final TokenCategory category; - - RustTokenId(int javaccKind, TokenCategory category) { - this.javaccKind = javaccKind; - this.category = category; - } - - public static RustTokenId get(int javaccKind) { - RustTokenId kind = LOOKUP[javaccKind]; - if (kind == null) { - throw new IllegalArgumentException("No TokenKind for constant: " + javaccKind); - } - return kind; - } - - @Override - public String primaryCategory() { - return category.getName(); - } -} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java new file mode 100644 index 0000000..06e20eb --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLanguageHierarchy.java @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2017 drrb + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.netbeans.RustLanguage; +import org.netbeans.spi.lexer.LanguageHierarchy; +import org.netbeans.spi.lexer.Lexer; +import org.netbeans.spi.lexer.LexerRestartInfo; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +public class AntlrRustLanguageHierarchy extends LanguageHierarchy { + + public static final AntlrRustLanguageHierarchy INSTANCE = new AntlrRustLanguageHierarchy(); + + final AntlrTokenIDs tokenIds; + public AntlrRustLanguageHierarchy() { + tokenIds = AntlrTokenIDs.forVocabulary(RustParser.VOCABULARY, RustAntlrLexer::categoryFor); + } + + @Override + protected Collection createTokenIds() { + return tokenIds.all(); + } + + @Override + protected Lexer createLexer(LexerRestartInfo info) { + return new RustAntlrLexer(info); + } + + @Override + protected String mimeType() { + return RustLanguage.MIME_TYPE; + } + + @Override + protected Map> createTokenCategories() { + Map> result = new HashMap<>(); + for (AntlrTokenID id : CommonRustTokenIDs.all()) { + Collection tokens = result.get(id.primaryCategory()); + if (tokens == null) { + tokens = new HashSet<>(); + result.put(id.primaryCategory(), tokens); + } + tokens.add(id); + } + return result; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java similarity index 69% rename from src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java rename to src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java index b8bab8b..40ec117 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/RustLexUtils.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrRustLexUtils.java @@ -14,57 +14,55 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -package com.github.drrb.rust.netbeans.parsing; +package com.github.drrb.rust.netbeans.parsing.antlr; +import com.github.drrb.rust.netbeans.parsing.*; import com.github.drrb.rust.netbeans.util.Option; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenId; import org.netbeans.api.lexer.TokenSequence; import org.netbeans.modules.csl.api.OffsetRange; import org.netbeans.modules.csl.spi.ParserResult; -import org.netbeans.modules.java.source.usages.DocumentUtil; - -import javax.swing.text.AbstractDocument; import javax.swing.text.BadLocationException; import javax.swing.text.Document; import java.util.logging.Level; import java.util.logging.Logger; -public class RustLexUtils { - private static final Logger LOG = Logger.getLogger(RustLexUtils.class.getName()); +public class AntlrRustLexUtils { + private static final Logger LOG = Logger.getLogger(AntlrRustLexUtils.class.getName()); - public TokenSequence getRustTokenSequence(Document doc, int offset) { + public TokenSequence getRustTokenSequence(Document doc, int offset) { TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc); - TokenSequence topLevelTokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence topLevelTokenSequence = tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); if (topLevelTokenSequence != null) { return topLevelTokenSequence; } - TokenSequence embeddedRustTokenSequence = getEmbeddedRustTokenSequence(tokenHierarchy, offset, true); + TokenSequence embeddedRustTokenSequence = getEmbeddedRustTokenSequence(tokenHierarchy, offset, true); if (embeddedRustTokenSequence != null) { return embeddedRustTokenSequence; } - TokenSequence embeddedRustTokenSequenceForwards = getEmbeddedRustTokenSequence(tokenHierarchy, offset, false); + TokenSequence embeddedRustTokenSequenceForwards = getEmbeddedRustTokenSequence(tokenHierarchy, offset, false); if (embeddedRustTokenSequenceForwards != null) { return embeddedRustTokenSequenceForwards; } try { LOG.warning("Couldn't get Rust token sequence for document. Falling back to lexing it ourselves."); - tokenHierarchy = TokenHierarchy.create(doc.getText(0, doc.getLength()), RustTokenId.language()); - return tokenHierarchy.tokenSequence(RustTokenId.language()); + tokenHierarchy = TokenHierarchy.create(doc.getText(0, doc.getLength()), AntlrRustLanguageHierarchy.INSTANCE.language()); + return tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); } catch (BadLocationException ex) { LOG.log(Level.WARNING, "Couldn't get Rust token sequence for document at all!", ex); return null; } } - private TokenSequence getEmbeddedRustTokenSequence(TokenHierarchy tokenHierarchy, int offset, boolean backwardBias) { + private TokenSequence getEmbeddedRustTokenSequence(TokenHierarchy tokenHierarchy, int offset, boolean backwardBias) { for (TokenSequence tokenSequence : tokenHierarchy.embeddedTokenSequences(offset, backwardBias)) { - if (tokenSequence.language() == RustTokenId.language()) { + if (tokenSequence.language() == AntlrRustLanguageHierarchy.INSTANCE.language()) { @SuppressWarnings("unchecked") - TokenSequence embeddedTokenSequence = (TokenSequence) tokenSequence; + TokenSequence embeddedTokenSequence = (TokenSequence) tokenSequence; return embeddedTokenSequence; } } @@ -77,17 +75,17 @@ public static Option getIdentifierAt(int caretOffset, ParserRes } public static Option getIdentifierAt(int caretOffset, TokenHierarchy tokenHierarchy) { - TokenSequence tokenSequence = tokenHierarchy.tokenSequence(RustTokenId.language()); + TokenSequence tokenSequence = tokenHierarchy.tokenSequence(AntlrRustLanguageHierarchy.INSTANCE.language()); Option tokenAtOffset = offsetTokenAt(caretOffset, tokenSequence); if (tokenAtOffset.isNot()) { return Option.none(); } - - if (tokenAtOffset.value().id() == RustTokenId.IDENTIFIER) { + TokenId identId = CommonRustTokenIDs.identifierTokenID(); + if (tokenAtOffset.value().id() == identId) { return tokenAtOffset; } else if (caretOffset > 0) { Option tokenBeforeOffset = offsetTokenAt(caretOffset - 1, tokenSequence); - if (tokenBeforeOffset.is() && tokenBeforeOffset.value().id() == RustTokenId.IDENTIFIER) { + if (tokenBeforeOffset.is() && tokenBeforeOffset.value().id() == identId) { return tokenBeforeOffset; } else { return Option.none(); @@ -97,7 +95,7 @@ public static Option getIdentifierAt(int caretOffset, TokenHier } } - private static Option offsetTokenAt(int caretPosition, TokenSequence tokenSequence) { + private static Option offsetTokenAt(int caretPosition, TokenSequence tokenSequence) { tokenSequence.move(caretPosition); if (tokenSequence.moveNext()) { return Option.is(OffsetRustToken.atCurrentLocation(tokenSequence)); diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java new file mode 100644 index 0000000..9380aa7 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrStreamAdapter.java @@ -0,0 +1,162 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.misc.Interval; +import org.netbeans.spi.lexer.LexerInput; + +public class AntlrStreamAdapter implements CharStream { + + private final String name; + private int index = 0; + private int mark = 0; + private final LexerInput input; + + public AntlrStreamAdapter(LexerInput input, String name) { + this.input = input; + this.name = name; + } + + @Override + public int index() { + return index; + } + + @Override + public int size() { + return -1; + } + + @Override + public String getSourceName() { + return name; + } + + @Override + public void consume() { + int character = read(); + if ( character == EOF ) { + backup( 1 ); + throw new IllegalStateException( "Attempting to consume EOF" ); + } + } + + @Override + public int LA(int lookaheadAmount) { + if ( lookaheadAmount < 0 ) { + return lookBack( -lookaheadAmount ); + } else if ( lookaheadAmount > 0 ) { + return lookAhead( lookaheadAmount ); + } else { + return 0; + } + } + + private int lookBack(int amount) { + backup( amount ); + int character = read(); + for ( int i = 1; i < amount; i++ ) { + read(); + } + return character; + } + + private int lookAhead(int amount) { + int character = 0; + for ( int i = 0; i < amount; i++ ) { + character = read(); + } + backup( amount ); + return character; + } + + @Override + public int mark() { + return ++mark; + } + + @Override + public void release(int marker) { + mark = marker; + mark--; + } + + @Override + public void seek(int index) { + if ( index < 0 ) { + throw new IllegalArgumentException( String.format( "Invalid index (%s < 0)", index ) ); + } + + if ( index < this.index ) { + backup( this.index - index ); + return; + } + while ( this.index < index ) { + consume(); + } + } + + private int read() { + int result = input.read(); + index++; + + if ( result == LexerInput.EOF ) { + return EOF; + } else { + return result; + } + } + + private void backup(int count) { + input.backup( count ); + index -= count; + } + + @Override + public String getText(Interval interval) { + int start = interval.a; + int stop = interval.b; + + if ( start < 0 || stop < start ) { + return ""; + } + + final int pos = this.index; + final int length = interval.length(); + final char[] data = new char[length]; + + seek( interval.a ); + int r = 0; + while ( r < length ) { + final int character = read(); + if ( character == EOF ) { + break; + } + + data[r] = (char) character; + r++; + } + seek( pos ); + + if ( r > 0 ) { + return new String( data, 0, r ); + } else { + return ""; + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java new file mode 100644 index 0000000..e57cf58 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenID.java @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Objects; +import org.netbeans.api.lexer.TokenId; + +/** + * Generic TokenId implementation for tokens from an Antlr 4 Vocabulary. + * + * @author Tim Boudreau + */ +public class AntlrTokenID implements TokenId { + + private final int tokenType; + private final String literalName; + private final String displayName; + private final String symbolicName; + private final String category; + + public static AntlrTokenID EOF = new AntlrTokenID(-1, "EOF", "EOF", "EOF", "other"); + + AntlrTokenID(int tokenType, String literalName, String displayName, + String symbolicName, String category) { + this.tokenType = tokenType; + this.literalName = literalName; + this.displayName = displayName; + this.symbolicName = symbolicName; + this.category = category; + } + + public String literalName() { + return literalName; + } + + public String symbolicName() { + return symbolicName(); + } + + public String displayName() { + return displayName; + } + + @Override + public String toString() { + return tokenType + "/" + (symbolicName == null ? "-" : "'" + symbolicName + "'") + + "/" + literalName + "/" + displayName; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof AntlrTokenID) { + AntlrTokenID other = (AntlrTokenID) o; + return other.tokenType == tokenType && + Objects.equals(other.literalName, literalName); + } + return false; + } + + @Override + public int hashCode() { + return ((tokenType + 1) * 51) * Objects.hashCode(literalName); + } + + @Override + public String name() { + if (symbolicName != null) { + return symbolicName; + } else if (literalName != null) { + return literalName; + } else if (displayName != null) { + return displayName; + } else { + return "????"; + } + } + + @Override + public int ordinal() { + return tokenType; + } + + @Override + public String primaryCategory() { + return category; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java new file mode 100644 index 0000000..4397509 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrTokenIDs.java @@ -0,0 +1,239 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.regex.Pattern; +import org.antlr.v4.runtime.Vocabulary; + +/** + * Generic support for generating a set of token IDs from any Antlr 4 + * Vocabulary - not specific to Rust. + * + * @author Tim Boudreau + */ +public class AntlrTokenIDs implements Iterable { + + private static Map mapping + = new WeakHashMap<>(); + + static synchronized AntlrTokenIDs forVocabulary(Vocabulary vocabulary, TokenCategorizer cat) { + AntlrTokenIDs result = mapping.get(vocabulary); + if (result == null) { + result = new AntlrTokenIDs(vocabulary, cat); + mapping.put(vocabulary, result); + } + return result; + } + + private Map byName = new HashMap<>(); + private final AntlrTokenID[] ids; + private final Map byCharacter = new HashMap<>(); + private final Map bySymbolicName = new HashMap<>(); + + AntlrTokenIDs(Vocabulary vocabulary, TokenCategorizer categorizer) { + int end = vocabulary.getMaxTokenType() + 1; + ids = new AntlrTokenID[end]; + for (int i = 0; i < end; i++) { + String displayName = stripSingleQuotes(vocabulary.getDisplayName(i)); + String symName = stripSingleQuotes(vocabulary.getSymbolicName(i)); + String litName = stripSingleQuotes(vocabulary.getLiteralName(i)); + String category = categorizer.categoryFor(i, displayName, + symName, litName); + ids[i] = new AntlrTokenID(i, litName, displayName, symName, category); + if (litName != null) { + byName.put(litName, ids[i]); + if (litName.length() == 1) { + byCharacter.put(litName.charAt(0), ids[i]); + } + } + if (symName != null) { + bySymbolicName.put(symName, ids[i]); + } + } + } + + public AntlrTokenID forSymbolicName(String name) { + if ("EOF".equals(name)) { + return AntlrTokenID.EOF; + } + return bySymbolicName.get(name); + } + + public AntlrTokenID forSymbol(char symbol) { + if (-1 == symbol) { + return AntlrTokenID.EOF; + } + return byCharacter.get(symbol); + } + + static String stripSingleQuotes(String s) { + if (s != null && s.length() > 1) { + char a = s.charAt(0); + char b = s.charAt(s.length() -1); + if (a == '\'' && b == '\'') { + s = s.substring(1, s.length()-1); + } + } + return s; + } + + public AntlrTokenID get(int tokenType) { + if (-1 == tokenType) { + return AntlrTokenID.EOF; + } + assert tokenType >= 0 && tokenType < ids.length : "Invalid token type " + tokenType; + return ids[tokenType]; + } + + public AntlrTokenID get(String literalName) { + if ("EOF".equals(literalName)) { + return AntlrTokenID.EOF; + } + AntlrTokenID result = byName.get(literalName); + if (result == null) { + throw new IllegalArgumentException("No token with symbolic name " + literalName); + } + return result; + } + + public int size() { + return ids.length; + } + + public List all() { + return Arrays.asList(ids); + } + + public Iterator iterator() { + return new ArrayIterator<>(ids); + } + + private static final class ArrayIterator implements Iterator { + + private final T[] arr; + private int index = -1; + + public ArrayIterator(T[] arr) { + this.arr = arr; + } + + @Override + public boolean hasNext() { + return index < arr.length - 1; + } + + @Override + public T next() { + return arr[++index]; + } + } + + public static void main(String[] args) { + TokenCategorizer cat = new TokenCategorizer() { + private final Pattern WORD = Pattern.compile("^[a-zA-Z]+$"); + @Override + public String categoryFor(int tokenType, String displayName, String symbolicName, String literalName) { + if (tokenType == 0) { + return "eof"; + } + System.out.println("DN " + displayName + " SN " + symbolicName + " LN " + literalName + " " + tokenType); + if (literalName != null && WORD.matcher(literalName).lookingAt()) { + return "keyword"; + } else if (literalName != null && literalName.length() == 1 && !Character.isAlphabetic(literalName.charAt(0))) { + switch(literalName.charAt(0)) { + case '*': + case '/': + case '%': + case '+': + case '-': + case '^': + case '|': + case '&': + return "operator"; + case '.' : + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case ',': + return "delimiter"; + case '<' : + case '>' : + return "comparisonOperator"; + case '=' : + return "assignmentOperator"; + } + return "symbol"; + } else if (literalName != null && literalName.length() == 2 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1))) { + switch(literalName) { + case "::" : + case "=>": + return "delimiter"; + case "+=" : + case "-=" : + case "/=" : + case "*=" : + case "%=" : + case "|=" : + case "&=" : + case "^=" : + return "assignmentOperator"; + case "==": + return "comparisonOperator"; + + } + return "symbol"; + } else if (literalName != null && literalName.length() == 3 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1)) && !Character.isAlphabetic(literalName.charAt(2))) { + switch(literalName) { + case "<<=" : + case ">>=" : + return "assignmentOperator"; + } + return "symbol"; + } else if (symbolicName != null) { + if (symbolicName.endsWith("Comment")) { + return "comment"; + } else if (symbolicName.endsWith("Lit")) { + return "literal"; + } + switch(symbolicName) { + case "Lifetime" : + return "keyword"; + case "Whitespace": + return "whitespace"; + case "Ident": + return "identifier"; + } + } + return "other"; + } + }; + AntlrTokenIDs ids = new AntlrTokenIDs(RustLexer.VOCABULARY, cat); + for (AntlrTokenID id : ids) { + System.out.println(id.primaryCategory() + "\t\t" + id.name()); + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java new file mode 100644 index 0000000..bee6aac --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/AntlrUtils.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Arrays; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +public final class AntlrUtils { + + private AntlrUtils() { + throw new AssertionError(); + } + + public static OffsetRange toOffsetRange(ParserRuleContext ctx) { + return new OffsetRange( + ctx.getStart().getStartIndex(), + ctx.getStop().getStopIndex() + 1); + } + + public static void print(ParserRuleContext ctx) { + StringBuilder sb = new StringBuilder("\n*******************************\n") + .append(ctx.getText()).append('\n'); + unwind(ctx, sb); + System.out.println(sb.toString()); + } + + public static String stringify(ParserRuleContext ctx) { + StringBuilder sb = new StringBuilder(); + unwind(ctx, sb); + return sb.toString(); + } + + private static void unwind(ParserRuleContext ctx, StringBuilder sb) { + unwind(ctx, 0, sb); + } + + private static void unwind(ParseTree ctx, int depth, StringBuilder sb) { + char[] ind = new char[depth * 2]; + Arrays.fill(ind, ' '); + sb.append(ind); + sb.append(ctx.getClass().getSimpleName()).append(" - ").append(ctx.getText()); + if (ctx instanceof ParserRuleContext) { + ParserRuleContext rule = (ParserRuleContext) ctx; + if (rule.children == null || rule.children.isEmpty()) { + sb.append(" with no children\n"); + } else { + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '\n') { + sb.append('\n'); + } + for (ParseTree c : rule.children) { + unwind(c, depth + 1, sb); + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '\n') { + sb.append('\n'); + } + } + } + } + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java new file mode 100644 index 0000000..81855e8 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/CommonRustTokenIDs.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrRustLanguageHierarchy.INSTANCE; + +/** + * + * @author Tim Boudreau + */ +public class CommonRustTokenIDs { + + public static AntlrTokenID stringLiteral() { + return forTokenType(RustLexer.StringLiteral); + } + + public static AntlrTokenID function() { + return forTokenType(RustLexer.Fn); + } + + public static AntlrTokenID eof() { + return AntlrTokenID.EOF; + } + + public static AntlrTokenID forSymbol(char symbol) { + return INSTANCE.tokenIds.forSymbol(symbol); + } + + public static AntlrTokenID forSymbolicName(String symbolicName) { + return INSTANCE.tokenIds.forSymbolicName(symbolicName); + } + + public static AntlrTokenID forLiteralName(String name) { + return INSTANCE.tokenIds.get(name); + } + + public static AntlrTokenID forTokenType(int id) { + return INSTANCE.tokenIds.get(id); + } + + public static AntlrTokenID whitespaceTokenID() { + return forTokenType(RustLexer.Whitespace); + } + + public static AntlrTokenID identifierTokenID() { + return forSymbolicName("Ident"); + } + + private static AntlrTokenID bang; + public static AntlrTokenID bang() { + return bang == null ? bang = forSymbol('!') : bang; + } + + private static AntlrTokenID semicolon; + public static AntlrTokenID semicolon() { + return semicolon == null ? semicolon = forTokenType(RustLexer.Semicolon) : semicolon; + } + + private static AntlrTokenID leftBrace; + public static AntlrTokenID leftBrace() { + return leftBrace == null ? leftBrace = forTokenType(RustLexer.LeftBrace) : leftBrace; + } + + private static AntlrTokenID rightBrace; + public static AntlrTokenID rightBrace() { + return rightBrace == null ? rightBrace = forTokenType(RustLexer.RightBrace) : rightBrace; + } + + private static AntlrTokenID leftParen; + public static AntlrTokenID leftParen() { + return leftParen == null ? leftParen = forTokenType(RustLexer.LeftParen) : leftParen; + } + + private static AntlrTokenID rightParen; + public static AntlrTokenID rightParen() { + return rightParen == null ? rightParen = forTokenType(RustLexer.RightParen) : rightParen; + } + + private static AntlrTokenID leftBracket; + public static AntlrTokenID leftBracket() { + return leftBracket== null ? leftBracket= forTokenType(RustLexer.LeftBracket) : leftBracket; + } + + private static AntlrTokenID rightBracket; + public static AntlrTokenID rightBracket() { + return rightBracket == null ? rightBracket= forTokenType(RustLexer.RightBracket) : rightBracket; + } + + private static AntlrTokenID leftAngleBracket; + public static AntlrTokenID leftAngleBracket() { + return leftAngleBracket== null ? leftAngleBracket= forTokenType(RustLexer.LeftAngleBracket) : leftAngleBracket; + } + + private static AntlrTokenID rightAngleBracket; + public static AntlrTokenID rightAngleBracket() { + return rightAngleBracket == null ? rightAngleBracket= forTokenType(RustLexer.RightAngleBracket) : rightAngleBracket; + } + + private static AntlrTokenID comma; + public static AntlrTokenID comma() { + return comma == null ? comma= forTokenType(RustLexer.Comma) : comma; + } + + public static Iterable all() { + return INSTANCE.tokenIds; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java new file mode 100644 index 0000000..ab2e53a --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/ErrImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.Severity; +import org.netbeans.modules.parsing.api.Snapshot; +import org.openide.filesystems.FileObject; + +/** + * Implementation of Error which wraps an ErrorNode encountered in a + * parse tree, distinguishable from syntax errors. + * + * @author Tim Boudreau + */ +final class ErrImpl implements Error { + + private final Severity severity; + private final String message; + private final FileObject file; + private final int start; + private final int end; + private final boolean line; + + ErrImpl(ErrorNode nd, Snapshot snaphsot) { + this(nd, Severity.FATAL, snaphsot); + } + + ErrImpl(ErrorNode nd, Severity severity, Snapshot snapshot) { + // Ensure we do not hold the snapshot or any objects from + // the parse, as these will be de-facto memory leaks + this.severity = severity; + message = nd.toString(); + file = snapshot.getSource().getFileObject(); + Interval interval = nd.getSourceInterval(); + int start = interval.a; + int end = interval.b; + if (start == -1) { + start = nd.getSymbol().getStartIndex(); + end = nd.getSymbol().getStopIndex() + 1; + } + // Negative starts will go boom, and start must be > end + // to produce highlighting + this.start = Math.max(0, start); + this.end = Math.max(1, end); + line = nd.getText() != null && !nd.getText().trim().contains("\n"); + } + + @Override + public int getStartPosition() { + return start; + } + + @Override + public int getEndPosition() { + return end; + } + + @Override + public boolean isLineError() { + return line; + } + + @Override + public String getDisplayName() { + return getDescription(); + } + + @Override + public String getDescription() { + return message; + } + + @Override + public String getKey() { + return getStartPosition() + ":" + getEndPosition() + ":" + getDescription(); + } + + @Override + public FileObject getFile() { + return file; + } + + @Override + public Severity getSeverity() { + return severity; + } + + @Override + public Object[] getParameters() { + // XXX what is this for? + return new Object[0]; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java new file mode 100644 index 0000000..c6ca8da --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAnalyzer.java @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustBaseVisitor; +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.antlr.RustParser.BlockContext; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrUtils.toOffsetRange; +import java.util.BitSet; +import java.util.EnumSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.antlr.v4.runtime.ANTLRErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.atn.ATNConfigSet; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.netbeans.modules.parsing.api.Snapshot; + +/** + * + * @author Tim Boudreau + */ +class RustAnalyzer extends RustBaseVisitor { + + private final RustParseInfo info; + private final Snapshot snapshot; + private final AtomicBoolean cancelled; + + RustAnalyzer(Snapshot snapshot, AtomicBoolean cancelled) { + this.info = new RustParseInfo(); + this.snapshot = snapshot; + this.cancelled = cancelled; + } + + @Override + public Void visit(ParseTree tree) { + if (cancelled.get()) { + return null; + } + return super.visit(tree); //To change body of generated methods, choose Tools | Templates. + } + + static RustParseInfo analyze(RustParser parser, Snapshot snapshot, AtomicBoolean cancelled) { + RustAnalyzer result = new RustAnalyzer(snapshot, cancelled); + parser.setErrorHandler(new ErrStrategy()); + parser.addErrorListener(new ErrorCapturer(snapshot, parser, result.info)); + parser.crate().accept(result); + return result.info; + } + + static boolean FULL_MESSAGES = Boolean.getBoolean("rust.antlr.full.messages"); + + static class ErrStrategy extends DefaultErrorStrategy { + + // Report full error messages for some tests + @Override + protected Token getMissingSymbol(Parser recognizer) { + Token oldRes = super.getMissingSymbol(recognizer); + if (true) { + return oldRes; + } + Token curr = recognizer.getCurrentToken(); +// System.out.println("oldRes type " + oldRes.getType() + " index " + oldRes.getTokenIndex()); + if (oldRes.getTokenIndex() == -1) { + IntervalSet expecting = getExpectedTokens(recognizer); + int expectedTokenType = Token.INVALID_TYPE; + if (!expecting.isNil()) { +// System.out.println("EXPECTING " + expecting.toString(recognizer.getVocabulary())); + expectedTokenType = expecting.getMinElement(); // get any element + } +// System.out.println("EXPECTED TOKEN TYPE " + recognizer.getVocabulary().getDisplayName(nextTokensState)); + String toInsert = null; + switch (curr.getText()) { + case "(": + toInsert = ")"; + break; + case "[": + toInsert = "]"; + break; + case "{": + toInsert = "}"; + break; + case "<": + toInsert = ">"; + break; + } + if (toInsert != null) { + System.out.println("CONJURE MISSING TOKEN " + toInsert + + " for " + curr.getText() + " index " + + curr.getTokenIndex() + " type " + curr.getType()); + + int start = curr.getStartIndex(); + int stop = curr.getStopIndex(); + return recognizer.getTokenFactory().create(new Pair( + curr.getTokenSource(), curr.getTokenSource().getInputStream()), + expectedTokenType, toInsert, + Token.DEFAULT_CHANNEL, + start, stop, + curr.getLine(), curr.getCharPositionInLine()); + } + } + + System.out.println("GET MISSING SYMBOL at " + recognizer.getCurrentToken() + + " super returns " + oldRes); + return oldRes; + } + + @Override + protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) { + Token offending = e.getOffendingToken(); + if (offending.getText().length() == 1 && !FULL_MESSAGES) { + recognizer.notifyErrorListeners(offending, "Unexpected symbol: '" + offending.getText() + "'", e); + } else { + super.reportNoViableAlternative(recognizer, e); + } + } + + @Override + protected void reportUnwantedToken(Parser recognizer) { + if (FULL_MESSAGES) { + super.reportUnwantedToken(recognizer); + return; + } + if (inErrorRecoveryMode(recognizer)) { + return; + } + + beginErrorCondition(recognizer); + + Token t = recognizer.getCurrentToken(); + String tokenName = getTokenErrorDisplay(t); + IntervalSet expecting = getExpectedTokens(recognizer); + String msg; + // Keep error messages to a reasonable size - if we're going + // to print out every keyword in the language, that doesn't + // help anyone + if (expecting.size() > 4) { + msg = "extraneous input " + tokenName; + } else { + msg = "extraneous input " + tokenName + " expecting " + + expecting.toString(recognizer.getVocabulary()); + } + recognizer.notifyErrorListeners(t, msg, null); + } + } + + static class ErrorCapturer implements ANTLRErrorListener { + + private final Snapshot snapshot; + private final RustParser parser; + private final RustParseInfo info; + + public ErrorCapturer(Snapshot snapshot, RustParser parser, RustParseInfo info) { + this.snapshot = snapshot; + this.parser = parser; + this.info = info; + } + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + org.antlr.v4.runtime.Token currentToken = parser.getCurrentToken(); + if (!FULL_MESSAGES) { + switch (msg) { + case "extraneous input ''": + msg = "Premature end of file"; + } + } + info.addError(new SyntaxError(currentToken, msg, snapshot.getSource().getFileObject())); + +// info.addError(new DefaultError(currentToken.toString(), "Syntax error", msg, +// snapshot.getSource().getFileObject(), +// currentToken.getStartIndex(), currentToken.getStopIndex() + 1, +// true, Severity.FATAL)); + } + + @Override + public void reportAmbiguity(Parser parser, DFA dfa, int i, int i1, boolean bln, BitSet bitset, ATNConfigSet atncs) { +// System.out.println("AE ambiguity at " + i + ":" + i1 + " " + dfa.toLexerString() +// + " " + parser.getCurrentToken()); +// String exp = parser.getExpectedTokens().toString(parser.getVocabulary()); +// System.out.println("EXPECTED: " + exp); + } + + @Override + public void reportAttemptingFullContext(Parser parser, DFA dfa, int i, int i1, BitSet bitset, ATNConfigSet atncs) { +// System.out.println("AE attempt full context at " + i + ":" + i1); + } + + @Override + public void reportContextSensitivity(Parser parser, DFA dfa, int i, int i1, int i2, ATNConfigSet atncs) { +// System.out.println("AE context sensitivity at " + i + ":" + i1); + } + + } + + @Override + public Void visitField(RustParser.FieldContext ctx) { + return super.visitField(ctx); + } + + @Override + public Void visitAttr(RustParser.AttrContext ctx) { + info.addSemanticRegion(RustElementKind.ATTR, toOffsetRange(ctx)); + info.addStructureItem(new RustStructureItemImpl(ctx.toString(), RustElementKind.ATTR, snapshot, ctx)); + return super.visitAttr(ctx); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public Void visitField_decl(RustParser.Field_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.FIELD); + if (id != null) { + info.addStructureItem(new RustStructureItemImpl(id, RustElementKind.FIELD, snapshot, ctx)); + } + return super.visitField_decl(ctx); + } + + @Override + public Void visitEnum_variant(RustParser.Enum_variantContext ctx) { + String id = findIdentifier(ctx, RustElementKind.ENUM_CONSTANT); + if (id != null) { + info.addStructureItem(new RustStructureItemImpl(id, RustElementKind.FIELD, snapshot, ctx)); + } + return super.visitEnum_variant(ctx); //To change body of generated methods, choose Tools | Templates. + } + + public static void unwind(ParserRuleContext ctx) { + AntlrUtils.print(ctx); + } + + @Override + public Void visitFn_head(RustParser.Fn_headContext ctx) { + String id = findIdentifier(ctx, RustElementKind.FUNCTION); + if (id != null) { + id = id.trim(); + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.FUNCTION, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitFn_head(ctx); + }); + } else { + super.visitFn_head(ctx); + } + return null; + } + + @Override + public Void visitLifetime_list(RustParser.Lifetime_listContext ctx) { + info.addSemanticRegion(RustElementKind.LIFETIME, toOffsetRange(ctx)); + return super.visitLifetime_list(ctx); + } + + @Override + public Void visitEnum_decl(RustParser.Enum_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.ENUM); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.ENUM, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitEnum_decl(ctx); + }); + } else { + super.visitEnum_decl(ctx); + } + return null; + } + + @Override + public Void visitType_decl(RustParser.Type_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.TYPE); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.TYPE, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitType_decl(ctx); + }); + } else { + super.visitType_decl(ctx); + } + return null; + } + + @Override + public Void visitImpl_block(RustParser.Impl_blockContext ctx) { + String id = findIdentifier(ctx, RustElementKind.IMPL); +// System.out.println("IMPL BLOCK FOUND ID " + id); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.IMPL, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitImpl_block(ctx); + }); + } else { + super.visitImpl_block(ctx); + } + return null; + } + + @Override + public Void visitStruct_decl(RustParser.Struct_declContext ctx) { + String id = findIdentifier(ctx.getParent(), RustElementKind.STRUCT); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.STRUCT, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitStruct_decl(ctx); + }); + } else { + super.visitStruct_decl(ctx); + } + return null; + } + + @Override + public Void visitMacro_tail(RustParser.Macro_tailContext ctx) { + return super.visitMacro_tail(ctx); + } + + @Override + public Void visitTrait_decl(RustParser.Trait_declContext ctx) { + String id = findIdentifier(ctx, RustElementKind.TRAIT); + if (id != null) { + RustStructureItemImpl item = new RustStructureItemImpl(id, RustElementKind.TRAIT, snapshot, ctx); + info.pushStructureItem(item, () -> { + super.visitTrait_decl(ctx); + }); + } else { + super.visitTrait_decl(ctx); + } + return null; + } + + private final IdentifierFinder idFinder = new IdentifierFinder(); + + private String findIdentifier(ParserRuleContext ctx, RustElementKind kind) { + String result = idFinder.find(ctx); + if (result != null) { + info.addSemanticRegion(idFinder.name, kind, toOffsetRange(idFinder.context), + idFinder.mutable, idFinder.visibility(), idFinder.statyc); + } + return result; + } + + static final class IdentifierFinder extends RustBaseVisitor { + + private String name; + private RustParser.IdentContext context; + private boolean mutable; + private boolean statyc; + private final Set visibility = EnumSet + .noneOf(RustVisibility.class); + + Set visibility() { + return EnumSet.copyOf(visibility); + } + + void reset() { + name = null; + context = null; + mutable = false; + statyc = false; + visibility.clear(); + } + + String find(ParserRuleContext ctx) { + reset(); + ctx.accept(this); + return this.name; + } + + @Override + public Void visit(ParseTree tree) { + if (name != null) { + return null; + } + return super.visit(tree); + } + + private void maybeAddVisibility(RustVisibility vis) { + if (vis != null) { + visibility.add(vis); + } + } + + boolean inVisitVisibility; + + @Override + public Void visitVisibility(RustParser.VisibilityContext ctx) { + inVisitVisibility = true; + try { + RustParser.Visibility_restrictionContext vr = ctx.visibility_restriction(); + if (vr != null) { + Token stop = vr.stop; + TokenSource src = vr.start.getTokenSource(); + CharStream in = src.getInputStream(); + for (Token tk = src.nextToken(); in.index() < in.size() && tk.getStopIndex() <= stop.getStopIndex(); tk = src.nextToken()) { + maybeAddVisibility(RustVisibility.forToken(tk)); + switch (tk.getText()) { + // XXX could look these up faster by token type, but + // because they aren't named in the grammar, they are, + // e.g. T__13, which is not a stable identifier. If + // we wind up patching the grammar, this is something + // to fix + case "pub": + case "crate": + case "super": + case "in": + } + } + } + return super.visitVisibility(ctx); + } finally { + inVisitVisibility = false; + } + } + + @Override + public Void visitIdent(RustParser.IdentContext ctx) { + if (inVisitVisibility) { + // "in" visibility will have an identifier which + // is not the one we want + return super.visitIdent(ctx); + } + if (name != null) { + return null; + } + assert name == null && context == null : "Already called with " + name; + name = ctx.getText(); + context = ctx; + // Don't call super - we're done visiting after we find + // the identifier + return null; + } + + @Override + public Void visitStatic_decl(RustParser.Static_declContext ctx) { + statyc = true; + return super.visitStatic_decl(ctx); + } + + @Override + public Void visitMut_or_const(RustParser.Mut_or_constContext ctx) { + if ("mut".equals(ctx.start.getText())) { + mutable = true; + } + return super.visitMut_or_const(ctx); + } + } + + @Override + public Void visitBlock(BlockContext ctx) { + info.addBlock(ctx); + return super.visitBlock(ctx); + } + + @Override + public Void visitTerminal(TerminalNode tn) { + return super.visitTerminal(tn); + } + + @Override + public Void visitErrorNode(ErrorNode en) { + if (en.getSourceInterval().a != -1) { + info.addError(new ErrImpl(en, snapshot)); + } + return super.visitErrorNode(en); + } + + @Override + public Void visitFn_decl(RustParser.Fn_declContext ctx) { +// unwind(ctx); + return super.visitFn_decl(ctx); //To change body of generated methods, choose Tools | Templates. + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java new file mode 100644 index 0000000..8a0d4a2 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrLexer.java @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import java.util.regex.Pattern; +import org.antlr.v4.runtime.CharStreams; +import org.netbeans.api.lexer.PartType; +import org.netbeans.api.lexer.Token; +import org.netbeans.spi.lexer.Lexer; +import org.netbeans.spi.lexer.LexerRestartInfo; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrLexer implements Lexer { + + public static com.github.drrb.rust.netbeans.rustbridge.RustLexer forString(String input) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + private final AntlrTokenIDs tokenIds; + private LexerRestartInfo info; + private RustLexer lexer; + public RustAntlrLexer() { + tokenIds = AntlrTokenIDs.forVocabulary(RustLexer.VOCABULARY, RustAntlrLexer::categoryFor); + } + + RustAntlrLexer(LexerRestartInfo info) { + tokenIds = AntlrTokenIDs.forVocabulary(RustLexer.VOCABULARY, RustAntlrLexer::categoryFor); + this.info = info; + lexer = new RustLexer( new AntlrStreamAdapter( info.input(), "RustAntlrLexer" ) ); +// lexer = new RustLexer(CharStreams.fromString(info.input().readText().toString())); + } + + public static RustLexer fromString(String s) { + return new RustLexer(CharStreams.fromString(s)); + } + + @Override + public Token nextToken() { + org.antlr.v4.runtime.Token antlrToken = lexer.nextToken(); + AntlrTokenID id; + if ( antlrToken.getType() == RustLexer.EOF && antlrToken.getStopIndex() < antlrToken.getStartIndex() ) { + return null; + } +// if ( info.input().readLength() < 1 ) { +// return null; // XXX eof? +// } + assert antlrToken.getType() <= RustLexer.VOCABULARY.getMaxTokenType(); + id = tokenIds.get( antlrToken.getType() ); + Token tok = info.tokenFactory().createToken( id, ( antlrToken.getStopIndex() + - antlrToken.getStartIndex() ) + 1, PartType.COMPLETE ); + return tok; + } + + @Override + public Object state() { + return null; + } + + @Override + public void release() { + lexer = null; + info = null; + } + + private static final Pattern WORD = Pattern.compile("^[a-zA-Z]+$"); + static String categoryFor(int tokenType, String displayName, String symbolicName, String literalName) { + if (tokenType == 0) { + return "eof"; + } + if (literalName != null && WORD.matcher(literalName).lookingAt()) { + return "keyword"; + } else if (literalName != null && literalName.length() == 1 && !Character.isAlphabetic(literalName.charAt(0))) { + switch (literalName.charAt(0)) { + case '*': + case '/': + case '%': + case '+': + case '-': + case '^': + case '|': + case '&': + return "operator"; + case '.': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': + case ',': + return "delimiter"; + case '<': + case '>': + return "comparisonOperator"; + case '=': + return "assignmentOperator"; + } + return "symbol"; + } else if (literalName != null && literalName.length() == 2 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1))) { + switch (literalName) { + case "::": + case "=>": + return "delimiter"; + case "+=": + case "-=": + case "/=": + case "*=": + case "%=": + case "|=": + case "&=": + case "^=": + return "assignmentOperator"; + case "==": + return "comparisonOperator"; + + } + return "symbol"; + } else if (literalName != null && literalName.length() == 3 && !Character.isAlphabetic(literalName.charAt(0)) && !Character.isAlphabetic(literalName.charAt(1)) && !Character.isAlphabetic(literalName.charAt(2))) { + switch (literalName) { + case "<<=": + case ">>=": + return "assignmentOperator"; + } + return "symbol"; + } else if (symbolicName != null) { + if (symbolicName.endsWith("Comment")) { + return "comment"; + } else if (symbolicName.endsWith("Lit")) { + return "literal"; + } + switch (symbolicName) { + case "Lifetime": + return "keyword"; + case "Whitespace": + return "whitespace"; + case "Ident": + return "identifier"; + } + } + return "other"; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java new file mode 100644 index 0000000..1119c18 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParser.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustLexer; +import com.github.drrb.rust.antlr.RustParser; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.swing.event.ChangeListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.netbeans.modules.parsing.api.Snapshot; +import org.netbeans.modules.parsing.api.Task; +import org.netbeans.modules.parsing.spi.ParseException; +import org.netbeans.modules.parsing.spi.Parser; +import org.netbeans.modules.parsing.spi.SourceModificationEvent; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrParser extends Parser { + private RustAntlrParserResult result; + private AtomicBoolean cancelled = new AtomicBoolean(); + + public RustParser parseString(String source) { + RustLexer lexer = new RustLexer(CharStreams.fromString(source)); + return new RustParser(new CommonTokenStream(lexer, 0)); + } + + @Override + public void parse(Snapshot snpsht, Task task, SourceModificationEvent sme) throws ParseException { + cancelled.set(false); + String source = snpsht.getText().toString(); + RustParser parser = parseString(source); + RustAntlrParserResult result = new RustAntlrParserResult(snpsht, parser, cancelled); + synchronized(this) { + this.result = result; + } +// System.out.println("PARSE RESULT " + result); + } + + @Override + public void cancel(CancelReason cr, SourceModificationEvent sme) { + System.out.println("cancelled because of " + cr); + cancel(); + } + + @Override + public void cancel() { + System.out.println("parse cancelled"); + cancelled.set(true); + RustAntlrParserResult result; + synchronized(this) { + result = this.result; + } + if (result != null) { + result.invalidate(); + } + } + + + @Override + public Result getResult(Task task) throws ParseException { + return result; + } + + @Override + public void addChangeListener(ChangeListener cl) { + // do nothing + } + + @Override + public void removeChangeListener(ChangeListener cl) { + // do nothing + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java new file mode 100644 index 0000000..229c7b2 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrParserResult.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.antlr.RustVisitor; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.spi.ParserResult; +import org.netbeans.modules.parsing.api.Snapshot; + +/** + * + * @author Tim Boudreau + */ +public final class RustAntlrParserResult extends ParserResult { + + private final RustParseInfo info; + private RustParser parser; + + public RustAntlrParserResult(Snapshot snapshot, RustParser parser, AtomicBoolean cancelled) { + super(snapshot); + this.parser = parser; + info = RustAnalyzer.analyze(parser, snapshot, cancelled); + } + + public String toString() { + return "RustAntlrParserResult {" + info + "}"; + } + + @SuppressWarnings("null") + public boolean accept(RustVisitor visitor) { + RustParser parserLocal; + synchronized(this) { + parserLocal = this.parser; + } + boolean result = parserLocal != null && parserLocal.crate() != null; + if (result) { + parserLocal.crate().accept(visitor); + } + return result; + } + + public List blocks() { + return info.blocks(); + } + + public List semanticRegions() { + return info.semanticRegions(); + } + + public List structureItems() { + return info.structureItems(); + } + + public RustParseInfo info() { + return info; + } + + @Override + public List getDiagnostics() { + return info.errors(); + } + + @Override + protected synchronized void invalidate() { + info.clear(); + parser = null; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java new file mode 100644 index 0000000..b98ecfc --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrSemanticAnalyzer.java @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2017 drrb + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.SemanticAnalyzer; +import org.netbeans.modules.parsing.spi.Scheduler; +import org.netbeans.modules.parsing.spi.SchedulerEvent; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * + */ +public class RustAntlrSemanticAnalyzer extends SemanticAnalyzer { +// private static final Logger LOG = Logger.getLogger(RustAntlrSemanticAnalyzer.class.getName()); + + private final Map> highlights = new HashMap<>(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + + @Override + public void run(RustAntlrParserResult result, SchedulerEvent event) { + try { + highlights.clear(); + cancelled.set(false); + for (RustSourceRegion region : result.info().semanticRegions()) { + System.out.println("highlight " + region); + highlights.put(region.range(), region.attributes()); + } + } catch (Exception e) { + e.printStackTrace(System.err); + if (e instanceof RuntimeException) { + throw ((RuntimeException) e); + } else { + throw new RuntimeException(e); + } + } + } + + @Override + public Map> getHighlights() { + return new HashMap<>(highlights); + } + + @Override + public int getPriority() { + return 0; + } + + @Override + public Class getSchedulerClass() { + return Scheduler.EDITOR_SENSITIVE_TASK_SCHEDULER; + } + + @Override + public void cancel() { + cancelled.set(true); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java new file mode 100644 index 0000000..98a1e70 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustAntlrStructureScanner.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.parsing.antlr.RustAntlrParserResult; +import static com.github.drrb.rust.netbeans.parsing.antlr.RustFoldTypeProvider.BLOCKS; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.StructureItem; +import org.netbeans.modules.csl.api.StructureScanner; +import org.netbeans.modules.csl.spi.ParserResult; + +/** + * + * @author Tim Boudreau + */ +public class RustAntlrStructureScanner implements StructureScanner { + + @Override + public List scan(ParserResult pr) { + return scan((RustAntlrParserResult) pr); + } + + @Override + public Map> folds(ParserResult pr) { + return folds((RustAntlrParserResult) pr); + } + + @Override + public Configuration getConfiguration() { + return new Configuration( true, true, 4 ); + } + + private Map> folds(RustAntlrParserResult pr) { + // Need to return a mutable list here for sorting + return Collections.singletonMap(BLOCKS, new ArrayList<>(pr.blocks())); + } + + private List scan(RustAntlrParserResult pr) { + // Need to return a mutable list here for sorting + return new ArrayList<>(pr.structureItems()); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java new file mode 100644 index 0000000..d482f35 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustElementKind.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.netbeans.modules.csl.api.ElementKind; + +/** + * + * @author Tim Boudreau + */ +public enum RustElementKind { + + STRUCT, + TRAIT, + TYPE, + FUNCTION, + ENUM, + FIELD, + TYPE_REFERENCE, + LIFETIME, + ENUM_CONSTANT, + ATTR, + IMPL + ; + + public boolean isStructural() { + switch(this) { + case TRAIT : + case TYPE : + case ENUM : + case FIELD : + case FUNCTION : + case ENUM_CONSTANT : + case IMPL : + return true; + default : + return false; + } + } + + /** + * Provides a rough mapping to NetBeans' ElementKind enum + * (which has no items to distingush type vs structure, + * for example. + * + * @return + */ + public ElementKind toElementKind() { + switch (this) { + // An imperfect mapping to say the least + case TRAIT : + return ElementKind.ATTRIBUTE; + case STRUCT: + return ElementKind.INTERFACE; + case TYPE: + case ENUM: + return ElementKind.CLASS; + case FIELD: + case ENUM_CONSTANT: + return ElementKind.FIELD; + case FUNCTION: + return ElementKind.METHOD; + case LIFETIME: + return ElementKind.RULE; + case TYPE_REFERENCE : + return ElementKind.GLOBAL; + case ATTR : + return ElementKind.ATTRIBUTE; + case IMPL : + return ElementKind.CLASS; + default: + throw new AssertionError(this); + } + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java new file mode 100644 index 0000000..8fd75eb --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustFoldTypeProvider.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.RustLanguage; +import java.util.Arrays; +import java.util.Collection; +import org.netbeans.api.editor.fold.FoldTemplate; +import org.netbeans.api.editor.fold.FoldType; +import org.netbeans.api.editor.mimelookup.MimeRegistration; +import org.netbeans.api.editor.mimelookup.MimeRegistrations; +import org.netbeans.spi.editor.fold.FoldTypeProvider; + +/** + * + * @author Tim Boudreau + */ +@MimeRegistrations({ + @MimeRegistration( + mimeType = RustLanguage.MIME_TYPE, + service = FoldTypeProvider.class, + position = 1488)}) +public class RustFoldTypeProvider implements FoldTypeProvider { + + public static final String COMMENTS = "comments"; + public static final String BLOCKS = "blocks"; + private static final FoldType BLOCK_FOLDS = FoldType.create(BLOCKS, BLOCKS, FoldTemplate.DEFAULT_BLOCK ); + private static final FoldType COMMENT_FOLDS = FoldType.create(COMMENTS, COMMENTS, FoldTemplate.DEFAULT_BLOCK ); + + @Override + @SuppressWarnings("unchecked") + public Collection getValues(Class type) { + return Arrays.asList( BLOCK_FOLDS, COMMENT_FOLDS ); + } + + @Override + public boolean inheritable() { + return true; + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java new file mode 100644 index 0000000..5817bd4 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustParseInfo.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.antlr.RustParser; +import com.github.drrb.rust.netbeans.parsing.antlr.RustVisibility; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +final class RustParseInfo { + + Set errors = new LinkedHashSet<>(); + Set blocks = new LinkedHashSet<>(); + private List structureItems = new ArrayList<>(); + private RustStructureItemImpl currStructureItem; + private final List semanticRegions = new LinkedList<>(); + private static final Set VISIBILITY_NA = EnumSet.noneOf(RustVisibility.class); + + void clear() { + errors.clear(); + blocks.clear(); + structureItems.clear(); + semanticRegions.clear(); + } + + void addSemanticRegion(RustElementKind kind, OffsetRange range) { + addSemanticRegion("", kind, range, false, VISIBILITY_NA, false); + } + + void addSemanticRegion(String name, RustElementKind kind, OffsetRange range, boolean mutable, Set visibility, boolean statyc) { + RustElementKind childOf = currStructureItem == null ? null : currStructureItem.kind; + semanticRegions.add(new SemanticRegion(kind, name, range, mutable, visibility, statyc || (kind == RustElementKind.FUNCTION && currStructureItem == null), childOf)); + } + + List semanticRegions() { + return semanticRegions; + } + + List structureItems() { + return structureItems; + } + + boolean hasSyntaxError; + + void addError(Error error) { + if (hasSyntaxError && !(error instanceof SyntaxError)) { + // not interested in more subtle errors from ErrNode if + // we already know the source has syntax errors + return; + } + hasSyntaxError |= error instanceof SyntaxError; + errors.add(error); + } + + void addStructureItem(RustStructureItemImpl item) { + List items = this.structureItems; + if (currStructureItem != null) { + items = currStructureItem.nested(); + item.setIn(currStructureItem.qName()); + } + items.add(item); + } + + void pushStructureItem(RustStructureItemImpl item, Runnable run) { + RustStructureItemImpl prev = currStructureItem; + addStructureItem(item); + currStructureItem = item; + try { + run.run(); + } finally { + currStructureItem = prev; + } + } + + void addBlock(RustParser.BlockContext ctx) { + blocks.add(new OffsetRange(ctx.getSourceInterval().a, ctx.getSourceInterval().b)); + } + + public List errors() { + return new ArrayList<>(errors); + } + + public List blocks() { + return new ArrayList<>(blocks); + } + + public String toString() { + StringBuilder sb = new StringBuilder("errors: ["); + for (Iterator it = errors.iterator(); it.hasNext();) { + Error err = it.next(); + sb.append(err.getDescription()).append('@').append(err.getStartPosition()) + .append(':').append(err.getEndPosition()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] structure: ["); + for (Iterator it = structureItems.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] blocks ["); + for (Iterator it = blocks.iterator(); it.hasNext();) { + OffsetRange range = it.next(); + sb.append(range.getStart()).append(":").append(range.getEnd()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("] semantic ["); + for (Iterator it = semanticRegions.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java new file mode 100644 index 0000000..846c2ef --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustSourceRegion.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.Optional; +import java.util.Set; +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +public interface RustSourceRegion { + + Set attributes(); + + Optional childOf(); + + boolean hasVisibility(RustVisibility vis); + + boolean isMutable(); + + boolean isStatic(); + + RustElementKind kind(); + + OffsetRange range(); + + Set visibility(); + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java new file mode 100644 index 0000000..7be4bca --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItem.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import java.util.List; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.api.StructureItem; + +/** + * + * @author Tim Boudreau + */ +public interface RustStructureItem extends StructureItem { + RustElementKind rustKind(); + + @Override + public List getNestedItems(); + + public OffsetRange range(); +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java new file mode 100644 index 0000000..72f33e7 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustStructureItemImpl.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.RustLanguage; +import static com.github.drrb.rust.netbeans.parsing.antlr.AntlrUtils.toOffsetRange; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import javax.swing.ImageIcon; +import org.antlr.v4.runtime.ParserRuleContext; +import org.netbeans.modules.csl.api.ElementHandle; +import org.netbeans.modules.csl.api.ElementKind; +import org.netbeans.modules.csl.api.HtmlFormatter; +import org.netbeans.modules.csl.api.Modifier; +import org.netbeans.modules.csl.api.OffsetRange; +import org.netbeans.modules.csl.spi.ParserResult; +import org.netbeans.modules.parsing.api.Snapshot; +import org.openide.filesystems.FileObject; + +/** + * + * @author Tim Boudreau + */ +final class RustStructureItemImpl implements ElementHandle, RustStructureItem { + + private final String name; + final RustElementKind kind; + List nested; + private final FileObject file; + private final OffsetRange range; + private String in = ""; + + RustStructureItemImpl(String name, RustElementKind kind, Snapshot snapshot, ParserRuleContext ctx) { + this.name = name; + this.kind = kind; + this.file = snapshot.getSource().getFileObject(); + this.range = toOffsetRange(ctx); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name).append(" ").append(kind).append(" @").append(range.getStart()).append(":").append(range.getEnd()); + if (!in.isEmpty()) { + sb.append(" in " + in); + } + if (nested != null && !nested.isEmpty()) { + sb.append(" children: ["); + for (Iterator it = nested.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(","); + } + } + sb.append("]"); + } + return sb.toString(); + } + + List nested() { + if (nested == null) { + nested = new ArrayList<>(5); + } + return nested; + } + + String qName() { + if (!in.isEmpty()) { + return in + "." + name; + } else { + return name; + } + } + + void setIn(String in) { + this.in = in; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getSortText() { + return getName(); + } + + @Override + public String getHtml(HtmlFormatter hf) { + return getName(); + } + + @Override + public ElementHandle getElementHandle() { + return this; + } + + public RustElementKind rustKind() { + return kind; + } + + @Override + public ElementKind getKind() { + return kind.toElementKind(); + } + + @Override + public Set getModifiers() { + return Collections.emptySet(); + } + + @Override + public boolean isLeaf() { + return nested == null; + } + + @Override + public List getNestedItems() { + if (nested == null) { + return Collections.emptyList(); + } + return nested; + } + + @Override + public long getPosition() { + return range.getStart(); + } + + @Override + public long getEndPosition() { + return range.getEnd(); + } + + public OffsetRange range() { + return range; + } + + @Override + public ImageIcon getCustomIcon() { + return null; + } + + @Override + public FileObject getFileObject() { + return this.file; + } + + @Override + public String getMimeType() { + return RustLanguage.MIME_TYPE; + } + + @Override + public String getIn() { + return in; + } + + @Override + public boolean signatureEquals(ElementHandle eh) { + return getName().equals(eh.getName()) && Objects.equals(eh.getIn(), getIn()); + } + + @Override + public OffsetRange getOffsetRange(ParserResult pr) { + return range; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof RustStructureItemImpl) { + RustStructureItemImpl other = (RustStructureItemImpl) o; + return file.equals(other.file) && qName().equals(other.qName()) && range.getStart() == other.range.getStart() && range.getEnd() == other.range.getEnd(); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(kind, name, file, range.getStart(), range.getEnd()); + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java new file mode 100644 index 0000000..f54c990 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/RustVisibility.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.Token; + +/** + * + * @author Tim Boudreau + */ + enum RustVisibility { + PUB("pub"), CRATE("crate"), SUPER("super"), IN("in"); + private final String stringValue; + + RustVisibility(String stringValue) { + this.stringValue = stringValue; + } + + public String toString() { + return stringValue; + } + + public static RustVisibility forToken(Token tk) { + switch (tk.getText()) { + case "pub": + return PUB; + case "crate": + return CRATE; + case "super": + return SUPER; + case "in": + return IN; + default: + return null; + } + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java new file mode 100644 index 0000000..f8ac988 --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SemanticRegion.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.github.drrb.rust.netbeans.parsing.antlr; + +import com.github.drrb.rust.netbeans.parsing.antlr.RustVisibility; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; +import org.netbeans.modules.csl.api.ColoringAttributes; +import org.netbeans.modules.csl.api.OffsetRange; + +/** + * + * @author Tim Boudreau + */ +final class SemanticRegion implements RustSourceRegion { + + public final RustElementKind kind; + public final String text; + public final OffsetRange range; + public final boolean mutable; + public final Set visibility; + public final boolean statyc; + private final RustElementKind childOf; + + SemanticRegion(RustElementKind kind, String text, OffsetRange range, boolean mutable, Set visibility, boolean statyc, RustElementKind childOf) { + assert range != null : "Range is null"; + assert kind != null : "Kind is null"; + this.kind = kind; + this.text = text; + this.range = range; + this.mutable = mutable; + this.visibility = visibility; + this.statyc = statyc; + this.childOf = childOf; + } + + public String toString() { + StringBuilder sb = new StringBuilder(kind.name()); + sb.append(" @").append(range.getStart()).append(':').append(range.getEnd()); + if (mutable) { + sb.append(" mut"); + } + if (statyc) { + sb.append(" static"); + } + if (!visibility.isEmpty()) { + sb.append(" visibility: ["); + for (Iterator it = visibility.iterator(); it.hasNext();) { + RustVisibility vis = it.next(); + sb.append(' ').append(vis); + } + sb.append(']'); + } + if (text != null && !text.isEmpty()) { + String txt = text.replaceAll("\t", "\\\\t").replaceAll("\n", "\\\\n"); + sb.append(" text: '").append(txt).append('\''); + } + if (childOf != null) { + sb.append (" under: ").append(childOf); + } + return sb.toString(); + } + + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof SemanticRegion) { + SemanticRegion other = (SemanticRegion) o; + return range.getStart() == other.range().getStart() && range.getEnd() == other.range().getEnd(); + } + return false; + } + + @Override + public RustElementKind kind() { + return kind; + } + + @Override + public Optional childOf() { + return Optional.ofNullable(childOf); + } + + @Override + public OffsetRange range() { + return range; + } + + @Override + public boolean isMutable() { + return mutable; + } + + @Override + public Set visibility() { + return visibility; + } + + @Override + public boolean isStatic() { + return statyc; + } + + @Override + public boolean hasVisibility(RustVisibility vis) { + return visibility != null && visibility.contains(vis); + } + + @Override + public Set attributes() { + // Pending - differentiate things like trait methods from + // struct/type methods using childOf and different colorings + Set result = EnumSet.noneOf(ColoringAttributes.class); + if (mutable) { + result.add(ColoringAttributes.GLOBAL); //XXX + } + if (hasVisibility(RustVisibility.PUB)) { + result.add(ColoringAttributes.PUBLIC); + } + if (hasVisibility(RustVisibility.CRATE)) { + result.add(ColoringAttributes.PACKAGE_PRIVATE); + } + if (hasVisibility(RustVisibility.SUPER)) { + result.add(ColoringAttributes.PROTECTED); + } + if (hasVisibility(RustVisibility.IN)) { + result.add(ColoringAttributes.CUSTOM3); + } + if (statyc) { + result.add(ColoringAttributes.STATIC); + } + switch (kind) { + case ENUM_CONSTANT: + result.add(ColoringAttributes.ENUM); + break; + case FIELD: + result.add(ColoringAttributes.FIELD); + break; + case FUNCTION: + result.add(ColoringAttributes.METHOD); + break; + case ENUM: + result.add(ColoringAttributes.CLASS); + break; + case TYPE: + result.add(ColoringAttributes.CLASS); + break; + case STRUCT: + result.add(ColoringAttributes.CLASS); + break; + case TRAIT: + result.add(ColoringAttributes.INTERFACE); + break; + case LIFETIME: + result.add(ColoringAttributes.CUSTOM1); + break; + case TYPE_REFERENCE: + result.add(ColoringAttributes.TYPE_PARAMETER_USE); + break; + case ATTR: + result.add(ColoringAttributes.ANNOTATION_TYPE); + break; + case IMPL: + result.add(ColoringAttributes.CLASS); + break; + default: + throw new AssertionError(kind); + } + return result; + } + +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java new file mode 100644 index 0000000..9fa345b --- /dev/null +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/SyntaxError.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2018 Tim Boudreau + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.github.drrb.rust.netbeans.parsing.antlr; + +import org.antlr.v4.runtime.Token; +import org.netbeans.modules.csl.api.Error; +import org.netbeans.modules.csl.api.Severity; +import org.openide.filesystems.FileObject; + +/** + * + * @author Tim Boudreau + */ +public class SyntaxError implements Error { + + private final AntlrTokenID type; + private final int startIndex; + private final int stopIndex; + private final int line; + private final int charPositionInLine; + private final int channel; + private final int tokenIndex; + private final String description; + private final FileObject fo; + private final String text; + + public SyntaxError(Token token, String description, FileObject fo) { + this.description = description; + this.type = CommonRustTokenIDs.forTokenType(token.getType()); + this.line = token.getLine(); + this.charPositionInLine = token.getCharPositionInLine(); + this.startIndex = token.getStartIndex(); + this.stopIndex = token.getStopIndex(); + this.channel = token.getChannel(); + this.tokenIndex = token.getTokenIndex(); + this.fo = fo; + this.text = token.getText(); + } + + public AntlrTokenID type() { + return type; + } + + @Override + public String toString() { + return "@line: " + line + ":" + charPositionInLine + + " (pos: " + startIndex + ":" + (stopIndex + 1) + ") tok " + + tokenIndex + " '" + text + "' (" + type + "): " + description; + } + + public int line() { + return line; + } + + public int charPositionInLine() { + return charPositionInLine; + } + + public int channel() { + return channel; + } + + public int tokenIndex() { + return tokenIndex; + } + + @Override + public String getDisplayName() { + return "Syntax error"; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getKey() { + return type.name() + + ":" + line + + ":" + charPositionInLine + + ":" + fo.getName(); + } + + @Override + public FileObject getFile() { + return fo; + } + + @Override + public int getStartPosition() { + return startIndex; + } + + @Override + public int getEndPosition() { + return stopIndex + 1; + } + + @Override + public boolean isLineError() { + return true; + } + + @Override + public Severity getSeverity() { + return Severity.FATAL; + } + + @Override + public Object[] getParameters() { + return new Object[0]; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (o == null) { + return false; + } else if (o instanceof SyntaxError) { + return getKey().equals(((SyntaxError) o).getKey()); + } + return false; + } + + @Override + public int hashCode() { + return ((line + 1) * (charPositionInLine * 7) + + (this.type.ordinal() + 1)) * + (51 * (fo.getName().hashCode() + 1)); + } +} diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java similarity index 63% rename from src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java rename to src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java index b3bc29c..260c89f 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/ParseUtil.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/antlr/TokenCategorizer.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2017 drrb + * Copyright (C) 2018 Tim Boudreau * * This program is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software @@ -14,16 +14,14 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -package com.github.drrb.rust.netbeans.parsing.javacc; +package com.github.drrb.rust.netbeans.parsing.antlr; -import org.netbeans.modules.csl.api.OffsetRange; - -public class ParseUtil { +/** + * + * @author Tim Boudreau + */ +public interface TokenCategorizer { - private ParseUtil() { - } + public String categoryFor(int tokenType, String displayName, String symbolicName, String literalName); - public static OffsetRange offsetRange(SimpleNode node) { - return new OffsetRange(node.jjtGetFirstToken().absoluteBeginPosition - 1, node.jjtGetLastToken().absoluteEndPosition - 1); - } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java index 5c85752..5a24af7 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStruct.java @@ -35,6 +35,10 @@ public class RustStruct { this.body = body; } + public String toString() { + return name + "@" + offsetRange.getStart() + ":" + offsetRange.getEnd() + " {" + body + "}"; + } + public String getName() { return name; } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java index a9944b3..7648c46 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructBody.java @@ -17,6 +17,7 @@ package com.github.drrb.rust.netbeans.parsing.index; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import org.netbeans.modules.csl.api.OffsetRange; @@ -31,7 +32,18 @@ public class RustStructBody { RustStructBody(OffsetRange offsetRange, List fields) { this.offsetRange = offsetRange; - this.fields = fields; + this.fields = Collections.unmodifiableList(fields); + } + + public String toString() { + StringBuilder sb= new StringBuilder(); + for (Iterator it=fields.iterator(); it.hasNext();) { + sb.append(it.next()); + if (it.hasNext()) { + sb.append(','); + } + } + return sb.toString(); } public OffsetRange getOffsetRange() { diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java index 96fb731..dfcac3e 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/index/RustStructField.java @@ -38,4 +38,8 @@ public String getName() { public OffsetRange getOffsetRange() { return offsetRange; } + + public String toString() { + return name + "@" + offsetRange.getStart() + ":" + offsetRange.getEnd(); + } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java deleted file mode 100644 index 0ceeb0c..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/JavaccCharStream.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -import org.netbeans.spi.lexer.LexerInput; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.UnsupportedEncodingException; - -public class JavaccCharStream implements CharStream { - - private LexerInput input; - - public int offset = 0; - private int tokenOffset = 0; - private boolean trackLineColumn = true; - - public JavaccCharStream(LexerInput input) { - this.input = input; - } - - public char BeginToken() throws IOException { - tokenOffset = 0; - return readChar(); - } - - public String GetImage() { - return input.readText(input.readLength() - tokenOffset, input.readLength()).toString(); - } - - public char[] GetSuffix(int len) { - if (len > input.readLength()) - throw new IllegalArgumentException(); - return input.readText(input.readLength() - len, input.readLength()).toString().toCharArray(); - } - - public void ReInit(Reader stream, int i, int i0) { - throw new UnsupportedOperationException("Not yet implemented"); - } - - public void ReInit(InputStream stream, String encoding, int i, int i0) throws UnsupportedEncodingException { - throw new UnsupportedOperationException("Not yet implemented"); - } - - public void backup(int i) { - offset -= i; - tokenOffset -= i; - tokenOffset = tokenOffset < 0 ? 0 : tokenOffset; - input.backup(i); - } - - public int getBeginColumn() { - return 0; - } - - public int getBeginLine() { - return 0; - } - - public int getEndColumn() { - return 0; - } - - public int getEndLine() { - return 0; - } - - public char readChar() throws IOException { - offset++; - tokenOffset++; - int result = input.read(); - if (result == LexerInput.EOF) { - if (tokenOffset > 1) { //todo: why? - backup(1); - } - throw new IOException("LexerInput EOF"); - } - return (char) result; - } - - @Override - public int getColumn() { - return 0; - } - - @Override - public int getLine() { - return 0; - } - - @Override - public void Done() { - - } - - @Override - public void setTabSize(int i) { - - } - - @Override - public int getTabSize() { - return 0; - } - - @Override - public boolean getTrackLineColumn() { - return trackLineColumn; - } - - @Override - public void setTrackLineColumn(boolean trackLineColumn) { - this.trackLineColumn = trackLineColumn; - } -} - diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java index 8884d9d..a06e156 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustToken.java @@ -16,96 +16,138 @@ */ package com.github.drrb.rust.netbeans.parsing.javacc; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; import org.netbeans.modules.csl.api.OffsetRange; -import java.util.LinkedList; -import java.util.List; -import static com.github.drrb.rust.netbeans.parsing.RustTokenId.EOF; - -public class RustToken extends Token { - private final RustTokenId enumKind; - - public RustToken(int kind, String image) { +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; +import java.util.Objects; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.TokenSource; +import org.netbeans.api.lexer.PartType; +import org.netbeans.api.lexer.Token; +import org.netbeans.api.lexer.TokenHierarchy; + +public class RustToken extends Token implements org.antlr.v4.runtime.Token { + private final AntlrTokenID enumKind; + private int kind; + private String image; + private final OffsetRange range; + public RustToken(int kind, String image, OffsetRange range) { this.kind = kind; - this.enumKind = RustTokenId.get(kind); + this.enumKind = CommonRustTokenIDs.forTokenType(kind); this.image = image; + this.range = range; } public boolean isEof() { - return enumKind == EOF; + return enumKind == CommonRustTokenIDs.eof(); } - public RustTokenId kind() { + public AntlrTokenID kind() { return enumKind; } - public RustTokenId id() { + public AntlrTokenID id() { return kind(); } - public RustToken specialToken() { - return (RustToken) specialToken; + @Override + public String toString() { + return enumKind + ": '" + image + "'"; + } + + public OffsetRange offsetRange() { + return range; + } + + @Override + public CharSequence text() { + return image; } - public boolean hasSpecialToken() { - return specialToken != null; + @Override + public boolean isCustomText() { + return !Objects.equals(image, enumKind.literalName()); } - public boolean hasNext() { - return next != null; + @Override + public int length() { + return range.getLength(); } - public RustToken next() { - return (RustToken) next; + @Override + public int offset(TokenHierarchy th) { + return range.getStart(); } - public boolean hasNextSpecialToken() { - return hasNext() && next().hasSpecialToken(); + @Override + public boolean isFlyweight() { + return false; } - public RustToken nextSpecialToken() { - return next().getEarliestSpecialToken(); + @Override + public PartType partType() { + return PartType.COMPLETE; } - public RustToken getEarliestSpecialToken() { - if (specialToken == null) { - return null; - } - Token token = this; - while (token.specialToken != null) { - token = token.specialToken; - } - return (RustToken) token; + @Override + public boolean hasProperties() { + return false; } @Override - public String toString() { - return enumKind + ": '" + image + "'"; + public Object getProperty(Object o) { + return null; } - public RustToken nextTokenMaybeSpecial() { - if (hasNextSpecialToken()) { - return nextSpecialToken(); - } else if (hasNext()) { - return next(); - } else { - return null; - } + @Override + public String getText() { + return image; } - public List withSpecialTokens() { - LinkedList thisWithSpecialTokens = new LinkedList<>(); - RustToken token = this; - do { - thisWithSpecialTokens.addFirst(token); - token = token.specialToken(); - } while (token != null); - return thisWithSpecialTokens; + @Override + public int getType() { + return kind; } - public OffsetRange offsetRange() { - return new OffsetRange(absoluteBeginPosition - 1, absoluteEndPosition - 1); + @Override + public int getLine() { + return 0; + } + + @Override + public int getCharPositionInLine() { + return 0; + } + + @Override + public int getChannel() { + return 0; + } + + @Override + public int getTokenIndex() { + return 0; + } + + @Override + public int getStartIndex() { + return range.getStart(); + } + + @Override + public int getStopIndex() { + return range.getEnd() -1; + } + + @Override + public TokenSource getTokenSource() { + return null; + } + + @Override + public CharStream getInputStream() { + return null; } } diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java deleted file mode 100644 index 95fec8c..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/RustTokenFactory.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -public class RustTokenFactory { - - public static Token newToken(int ofKind, String tokenImage) { - return new RustToken(maybeTranslateSubkind(ofKind), tokenImage); - } - - private static int maybeTranslateSubkind(int kind) { - switch(kind) { - case RustParserConstants.RAW_STRING_LITERAL_0: - case RustParserConstants.RAW_STRING_LITERAL_1: - case RustParserConstants.RAW_STRING_LITERAL_2: - case RustParserConstants.RAW_STRING_LITERAL_3: - return RustParserConstants.RAW_STRING_LITERAL; - case RustParserConstants.RAW_BYTE_STRING_LITERAL_0: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_1: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_2: - case RustParserConstants.RAW_BYTE_STRING_LITERAL_3: - return RustParserConstants.RAW_BYTE_STRING_LITERAL; - default: - return kind; - } - } -} - diff --git a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java b/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java deleted file mode 100644 index 2e24ea7..0000000 --- a/src/main/java/com/github/drrb/rust/netbeans/parsing/javacc/SimpleCharStream.java +++ /dev/null @@ -1,500 +0,0 @@ -/** - * Copyright (C) 2017 drrb - * - * This program is free software: you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later - * version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program. If not, see . - */ -/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 7.0 */ -/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ -package com.github.drrb.rust.netbeans.parsing.javacc; - -/** - * An implementation of interface CharStream, where the stream is assumed to - * contain only ASCII characters (without unicode processing). - */ - -public class SimpleCharStream implements CharStream -{ - protected int totalCharsRead = 0; - protected int absoluteTokenBegin = 0; - public final int getAbsoluteTokenBegin() { - return absoluteTokenBegin; - } - -/** Whether parser is static. */ - public static final boolean staticFlag = false; - int bufsize; - int available; - int tokenBegin; -/** Position in buffer. */ - public int bufpos = -1; - protected int bufline[]; - protected int bufcolumn[]; - - protected int column = 0; - protected int line = 1; - - protected boolean prevCharIsCR = false; - protected boolean prevCharIsLF = false; - - protected java.io.Reader inputStream; - - protected char[] buffer; - protected int maxNextCharInd = 0; - protected int inBuf = 0; - protected int tabSize = 1; - protected boolean trackLineColumn = true; - - public void setTabSize(int i) { tabSize = i; } - public int getTabSize() { return tabSize; } - - - - protected void ExpandBuff(boolean wrapAround) - { - char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; - - try - { - if (wrapAround) - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); - bufcolumn = newbufcolumn; - - maxNextCharInd = (bufpos += (bufsize - tokenBegin)); - } - else - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - buffer = newbuffer; - - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - bufline = newbufline; - - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - bufcolumn = newbufcolumn; - - maxNextCharInd = (bufpos -= tokenBegin); - } - } - catch (Throwable t) - { - throw new Error(t.getMessage()); - } - - - bufsize += 2048; - available = bufsize; - tokenBegin = 0; - } - - protected void FillBuff() throws java.io.IOException - { - if (maxNextCharInd == available) - { - if (available == bufsize) - { - if (tokenBegin > 2048) - { - bufpos = maxNextCharInd = 0; - available = tokenBegin; - } - else if (tokenBegin < 0) - bufpos = maxNextCharInd = 0; - else - ExpandBuff(false); - } - else if (available > tokenBegin) - available = bufsize; - else if ((tokenBegin - available) < 2048) - ExpandBuff(true); - else - available = tokenBegin; - } - - int i; - try { - if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1) - { - inputStream.close(); - throw new java.io.IOException(); - } - else - maxNextCharInd += i; - return; - } - catch(java.io.IOException e) { - --bufpos; - backup(0); - if (tokenBegin == -1) - tokenBegin = bufpos; - throw e; - } - } - -/** Start. */ - public char BeginToken() throws java.io.IOException - { - tokenBegin = -1; - char c = readChar(); - tokenBegin = bufpos; - absoluteTokenBegin = totalCharsRead; - - return c; - } - - protected void UpdateLineColumn(char c) - { - column++; - - if (prevCharIsLF) - { - prevCharIsLF = false; - line += (column = 1); - } - else if (prevCharIsCR) - { - prevCharIsCR = false; - if (c == '\n') - { - prevCharIsLF = true; - } - else - line += (column = 1); - } - - switch (c) - { - case '\r' : - prevCharIsCR = true; - break; - case '\n' : - prevCharIsLF = true; - break; - case '\t' : - column--; - column += (tabSize - (column % tabSize)); - break; - default : - break; - } - - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - } - -/** Read a character. */ - public char readChar() throws java.io.IOException - { - if (inBuf > 0) - { - --inBuf; - - if (++bufpos == bufsize) - bufpos = 0; - - totalCharsRead++; - return buffer[bufpos]; - } - - if (++bufpos >= maxNextCharInd) - FillBuff(); - - totalCharsRead++; - char c = buffer[bufpos]; - - UpdateLineColumn(c); - return c; - } - - @Deprecated - /** - * @deprecated - * @see #getEndColumn - */ - - public int getColumn() { - return bufcolumn[bufpos]; - } - - @Deprecated - /** - * @deprecated - * @see #getEndLine - */ - - public int getLine() { - return bufline[bufpos]; - } - - /** Get token end column number. */ - public int getEndColumn() { - return bufcolumn[bufpos]; - } - - /** Get token end line number. */ - public int getEndLine() { - return bufline[bufpos]; - } - - /** Get token beginning column number. */ - public int getBeginColumn() { - return bufcolumn[tokenBegin]; - } - - /** Get token beginning line number. */ - public int getBeginLine() { - return bufline[tokenBegin]; - } - -/** Backup a number of characters. */ - public void backup(int amount) { - - inBuf += amount; - totalCharsRead -= amount; - if ((bufpos -= amount) < 0) - bufpos += bufsize; - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.Reader dstream) - { - this(dstream, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { - inputStream = dstream; - line = startline; - column = startcolumn - 1; - - if (buffer == null || buffersize != buffer.length) - { - available = bufsize = buffersize; - buffer = new char[buffersize]; - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; - } - prevCharIsLF = prevCharIsCR = false; - tokenBegin = inBuf = maxNextCharInd = 0; - bufpos = -1; - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.Reader dstream) - { - ReInit(dstream, 1, 1, 4096); - } - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, 1, 1, 4096); - } - - /** Constructor. */ - public SimpleCharStream(java.io.InputStream dstream) - { - this(dstream, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, 1, 1, 4096); - } - - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream) - { - ReInit(dstream, 1, 1, 4096); - } - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, startline, startcolumn, 4096); - } - /** Reinitialise. */ - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); - } - /** Get token literal value. */ - public String GetImage() - { - if (bufpos >= tokenBegin) - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); - else - return new String(buffer, tokenBegin, bufsize - tokenBegin) + - new String(buffer, 0, bufpos + 1); - } - - /** Get the suffix. */ - public char[] GetSuffix(int len) - { - char[] ret = new char[len]; - - if ((bufpos + 1) >= len) - System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); - else - { - System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, - len - bufpos - 1); - System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); - } - - return ret; - } - - /** Reset buffer when finished. */ - public void Done() - { - buffer = null; - bufline = null; - bufcolumn = null; - } - - /** - * Method to adjust line and column numbers for the start of a token. - */ - public void adjustBeginLineColumn(int newLine, int newCol) - { - int start = tokenBegin; - int len; - - if (bufpos >= tokenBegin) - { - len = bufpos - tokenBegin + inBuf + 1; - } - else - { - len = bufsize - tokenBegin + bufpos + 1 + inBuf; - } - - int i = 0, j = 0, k = 0; - int nextColDiff = 0, columnDiff = 0; - - while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) - { - bufline[j] = newLine; - nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; - bufcolumn[j] = newCol + columnDiff; - columnDiff = nextColDiff; - i++; - } - - if (i < len) - { - bufline[j] = newLine++; - bufcolumn[j] = newCol + columnDiff; - - while (i++ < len) - { - if (bufline[j = start % bufsize] != bufline[++start % bufsize]) - bufline[j] = newLine++; - else - bufline[j] = newLine; - } - } - - line = bufline[j]; - column = bufcolumn[j]; - } - public boolean getTrackLineColumn() { return trackLineColumn; } - public void setTrackLineColumn(boolean tlc) { trackLineColumn = tlc; } -} -/* JavaCC - OriginalChecksum=b094525db5a9a7246ced7b3d7fc11c7f (do not edit this line) */ diff --git a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java index cb4ab56..7f3380a 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java +++ b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustLexer.java @@ -16,7 +16,7 @@ */ package com.github.drrb.rust.netbeans.rustbridge; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; /** * @@ -28,7 +28,7 @@ public class RustLexer { @Override public RustToken.ByValue nextToken() { RustToken.ByValue token = new RustToken.ByValue(); - token.type = RustTokenId.EOF.ordinal(); + token.type = CommonRustTokenIDs.eof().ordinal(); return token; } diff --git a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java index d6c4493..344e210 100644 --- a/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java +++ b/src/main/java/com/github/drrb/rust/netbeans/rustbridge/RustToken.java @@ -16,10 +16,12 @@ */ package com.github.drrb.rust.netbeans.rustbridge; -import com.github.drrb.rust.netbeans.parsing.RustTokenId; +import com.github.drrb.rust.netbeans.parsing.antlr.AntlrTokenID; +import com.github.drrb.rust.netbeans.parsing.antlr.CommonRustTokenIDs; import com.sun.jna.Structure; import static java.util.Arrays.asList; import java.util.List; +import org.antlr.v4.runtime.Token; /** * @@ -39,12 +41,22 @@ public static class ByValue extends RustToken implements Structure.ByValue { public int endChar; public int type; + public static RustToken of(Token tok) { + RustToken result = new RustToken(); + result.startLine = tok.getLine(); + result.startCol = tok.getCharPositionInLine(); + result.type = tok.getType(); + result.endCol = tok.getCharPositionInLine() + ((tok.getStopIndex() + 1) - tok.getStartIndex()); + result.endLine = tok.getLine(); + return result; + } + boolean isEof() { - return getType() == RustTokenId.EOF; + return getType() == CommonRustTokenIDs.eof(); } - public RustTokenId getType() { - return RustTokenId.values()[type]; + public AntlrTokenID getType() { + return CommonRustTokenIDs.forTokenType(type); } public int length() { diff --git a/src/main/nbm/manifest.mf b/src/main/nbm/manifest.mf index cb4d374..68dd5c9 100644 --- a/src/main/nbm/manifest.mf +++ b/src/main/nbm/manifest.mf @@ -1,3 +1,4 @@ Manifest-Version: 1.0 OpenIDE-Module-Layer: com/github/drrb/rust/netbeans/layer.xml OpenIDE-Module-Localizing-Bundle: com/github/drrb/rust/netbeans/Bundle.properties +OpenIDE-Module-Install: com/github/drrb/rust/netbeans/Installer.class diff --git a/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml b/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml index a6b7392..81a5037 100644 --- a/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml +++ b/src/main/resources/com/github/drrb/rust/netbeans/FontAndColors.xml @@ -28,8 +28,10 @@ + + - + diff --git a/src/test/data/index/project/struct/src/main.rs b/src/test/data/index/project/struct/src/main.rs index 54f3fc4..92cf826 100644 --- a/src/test/data/index/project/struct/src/main.rs +++ b/src/test/data/index/project/struct/src/main.rs @@ -1,4 +1,4 @@ struct Person { name: String, age: usize, -} \ No newline at end of file +} diff --git a/src/test/data/parse/errors/errors_in_items.rs.errors b/src/test/data/parse/errors/errors_in_items.rs.errors index 196dd27..22b8206 100644 --- a/src/test/data/parse/errors/errors_in_items.rs.errors +++ b/src/test/data/parse/errors/errors_in_items.rs.errors @@ -1,104 +1,6 @@ -[rust.parse.message] 31-32:Parse error ; Encountered " "}" "} "" at line 3, column 1. -Was expecting one of: - ... - ... - ... - ... - ... - ... - ... - "::" ... - "(" ... - "-" ... - "*" ... - "&" ... - "!" ... - "false" ... - "for" ... - "if" ... - "loop" ... - "return" ... - "true" ... - "while" ... - ... -