diff --git a/.gitignore b/.gitignore index 5d5bac5..b0bf39d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +.idea build dist javalib/mirah-complete_parsersupport.jar javalib/mirahc.jar +*.iml +out diff --git a/src/DubyBootstrap.aj b/src/DubyBootstrap.aj deleted file mode 100644 index 0fd5da4..0000000 --- a/src/DubyBootstrap.aj +++ /dev/null @@ -1,46 +0,0 @@ -import java.util.ArrayList; -import java.util.List; -import mirah.lang.ast.Node; -import mirah.lang.ast.NodeScanner; -import mirah.lang.ast.NodeImpl; - -/* To compile the new AST with duby extensions: - * ajc -1.5 -inpath dist/mirah-parser.jar \ - * -outjar dist/mirah-parser_with_duby.jar \ - * -classpath ../mirah/javalib/mirah-bootstrap.jar:/Developer/aspectj1.6/lib/aspectjrt.jar \ - * src/DubyBootstrap.aj - */ - -class ChildCollector extends NodeScanner { - private ArrayList children = new ArrayList(); - - @Override - public boolean enterDefault(Node node, Object arg) { - if (node == arg) { - return true; - } else { - children.add(node); - return false; - } - } - - @Override - public Object enterNullChild(Object arg){ - children.add(null); - return null; - } - - public ArrayList children() { - return children; - } -} - -aspect DubyBootsrap { - declare parents: Node extends duby.lang.compiler.Node; - - public List NodeImpl.child_nodes() { - ChildCollector c = new ChildCollector(); - c.scan(this, this); - return c.children(); - } -} \ No newline at end of file diff --git a/src/mirah/impl/MirahLexer.java b/src/mirah/impl/MirahLexer.java new file mode 100644 index 0000000..f2c20f8 --- /dev/null +++ b/src/mirah/impl/MirahLexer.java @@ -0,0 +1,1365 @@ +/* + Copyright (c) 2010 The Mirah project authors. All Rights Reserved. + All contributing project authors may be found in the NOTICE file. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package mirah.impl; + +import mirahparser.impl.Tokens; +import mmeta.BaseParser; +import mmeta.BaseParser.Token; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.ListIterator; +import java.util.logging.Logger; +public class MirahLexer { + + private static final Logger logger = Logger.getLogger(MirahLexer.class.getName()); + + private static final int EOF = -1; + + public interface Input { + int pos(); + int read(); + boolean consume(char c); + boolean consume(String s); + void backup(int amount); + void skip(int amount); + boolean hasNext(); + void consumeLine(); + int peek(); + int finishCodepoint(); + CharSequence readBack(int length); + } + + public static class StringInput implements Input { + private int pos = 0; + private final String string; + private final char[] chars; + private final int end; + + public StringInput(String string, char[] chars) { + this.string = string; + this.chars = chars; + this.end = chars.length; + } + + public int pos() { + return pos; + } + + public int read() { + if (pos >= end) { + pos = end + 1; + return EOF; + } + return chars[pos++]; + } + + public boolean consume(char c) { + if (read() == c) { + return true; + } + --pos; + return false; + } + + public boolean consume(String s) { + if (string.startsWith(s, pos)) { + pos += s.length(); + return true; + } + return false; + } + + public void backup(int amount) { + pos -= amount; + } + + public void skip(int amount) { + if (pos < end - amount) { + pos += amount; + } else { + pos = end; + } + } + + public boolean hasNext() { + return pos < end; + } + + public void consumeLine() { + pos = string.indexOf('\n', pos); + if (pos == -1) { + pos = end; + } + } + + public int peek() { + int result = read(); + --pos; + return result; + } + + public int finishCodepoint() { + int size = 1; + if (pos < end - 1) { + ++pos; + ++size; + } + return string.codePointAt(pos - size); + } + + public CharSequence readBack(int length) { + return string.substring(pos - length, pos); + } + } + + protected interface Lexer { + Tokens skipWhitespace(MirahLexer l, Input i); + Tokens lex(MirahLexer l, Input i); + } + private static abstract class BaseLexer implements Lexer { + @Override + public Tokens skipWhitespace(MirahLexer l, Input i) { return null; } + } + protected static class State implements Cloneable { + public final Lexer lexer; + public final State previous; + public final boolean justOnce; + public int braceDepth; + public final LinkedList hereDocs = new LinkedList(); + + public State(State previous, Lexer lexer, boolean justOnce) { + this.previous = previous; + this.lexer = lexer; + this.justOnce = justOnce; + } + + public State(State previous, Lexer lexer) { + this(previous, lexer, false); + } + + public void lbrace() { + braceDepth += 1; + } + public void rbrace(MirahLexer ml) { + braceDepth -= 1; + if (braceDepth == -1) { + ml.popState(); + } + } + + public State clone() { + State clone = new State(previous == null ? null : previous.clone(), lexer, justOnce); + clone.braceDepth = braceDepth; + clone.hereDocs.addAll(hereDocs); + return clone; + } + } + + private static class CombinedState { + private final State state; + private final boolean isBEG; + private final boolean isARG; + private final boolean isEND; + private final boolean spaceSeen; + + public CombinedState(MirahLexer l) { + state = l.state.clone(); + isBEG = l.isBEG(); + isARG = l.isARG(); + isEND = l.isEND(); + spaceSeen = l.spaceSeen; + } + } + + private static class SStringLexer extends BaseLexer { + private boolean isEscape(Input i) { + return i.consume('\'') || i.consume('\\'); + } + + @Override + public Tokens lex(MirahLexer l, Input i) { + int c0 = i.read(); + switch (c0) { + case '\'': + l.popState(); + return Tokens.tSQuote; + case '\\': + if (isEscape(i)) { + return Tokens.tEscape; + } + } + readRestOfString(i); + return Tokens.tStringContent; + } + + private void readRestOfString(Input i) { + int c = 0; + for (c = i.read(); c != EOF;c = i.read()) { + if (c == '\'') { + i.backup(1); + break; + } else if (c == '\\' && isEscape(i)) { + i.backup(2); + break; + } + } + if ( c == EOF ){ + i.backup(1); + } + } + } + + private static class DStringLexer extends BaseLexer { + @Override + public Tokens lex(MirahLexer l, Input i) { + int c = i.read(); + if (isEndOfString(c)) { + l.popState(); + return readEndOfString(i); + } + switch (c) { + case '\\': + readEscape(i); + return Tokens.tEscape; + case '#': + int c2 = i.read(); + if (c2 == '{') { + l.pushState(l.state.previous.lexer); + return Tokens.tStrEvBegin; + } else if (c2 == '@') { + i.backup(1); + l.pushForOneToken(l.state.previous.lexer); + return Tokens.tStrEvBegin; + } + i.backup(1); + } + readRestOfString(i); + return Tokens.tStringContent; + } + public Tokens readEndOfString(Input i) { + return Tokens.tDQuote; + } + public boolean isEndOfString(int c) { + return c == '"'; + } + private void readEscape(Input i) { + int c = i.read(); + switch (c) { + case 'x': + i.skip(2); + return; + case 'u': + i.skip(4); + return; + case 'U': + i.skip(8); + return; + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': + int c1 = i.read(); + int c2 = i.read(); + if (c1 >= '0' && c1 <= '7' && c2 >= '0' && c2 <= '7') { + return; + } + i.backup(2); + default: + if (c >= 0xD800 && c <= 0xDFFF) { + i.skip(1); + } + } + } + + private void readRestOfString(Input i) { + int c = 0; + for (c = i.read(); c != EOF; c = i.read()) { + if (isEndOfString(c)) { + i.backup(1); + break; + } else if (c == '#') { + int c2 = i.read(); + if (c2 == '{' || c2 == '@') { + i.backup(2); + break; + } else { + i.backup(1); + } + } else if (c == '\\') { + i.backup(1); + break; + } + } + if ( c == EOF ){ + i.backup(1); + } + } + } + + private static class RegexLexer extends DStringLexer { + @Override + public boolean isEndOfString(int c) { + return c == '/'; + } + + @Override + public Tokens readEndOfString(Input i) { + for (int c = i.read(); c != EOF; c = i.read()) { + if (!Character.isLetter(c)){ + break; + } + } + i.backup(1); + return Tokens.tRegexEnd; + } + } + + private static class HereDocLexer extends BaseLexer { + private final String marker; + private final boolean allowIndented; + private final boolean allowStrEv; + + public HereDocLexer(String marker, boolean allowIndented, boolean allowStrEv) { + this.marker = marker; + this.allowIndented = allowIndented; + this.allowStrEv = allowStrEv; + } + + @Override + public Tokens lex(MirahLexer l, Input i) { + if (readMarker(i, true)) { + l.popState(); + return Tokens.tHereDocEnd; + } + if (allowStrEv && i.consume('#')) { + if (i.consume('{') || i.consume('@')) { + return readStrEv(l, i); + } + } + for (int c = i.read();c != EOF; c = i.read()) { + if (c == '\n') { + if (readMarker(i, false)) { + return Tokens.tStringContent; + } + } else if (allowStrEv && c == '#') { + if (i.consume('{') || i.consume('@')) { + i.backup(2); + return Tokens.tStringContent; + } + } + } + i.backup(1); + return Tokens.tStringContent; + } + + private boolean readMarker(Input i, boolean consume) { + int size = 0; + int c = i.read(); + if (allowIndented) { + while (" \t\r\f\u000b".indexOf(c) != -1) { + size += 1; + c = i.read(); + } + } + i.backup(1); + if (i.consume(marker)) { + size += marker.length(); + if (i.hasNext() && i.peek() != '\n') { + i.backup(size); + return false; + } + if (!consume) { + i.backup(size); + } + return true; + } + i.backup(size); + return false; + } + + private Tokens readStrEv(MirahLexer l, Input i) { + i.backup(1); + if (i.consume('{')) { + l.pushState(l.state.previous.lexer); + return Tokens.tStrEvBegin; + } else if (i.peek() == '@') { + l.pushForOneToken(l.state.previous.lexer); + return Tokens.tStrEvBegin; + } else { + return Tokens.tUNKNOWN; + } + } + } + + private static class StandardLexer implements Lexer { + + public Tokens lex(MirahLexer l, Input i) { + Tokens type = processFirstChar(l, i); + type = checkKeyword(type, i); + return readRestOfToken(type, l, i); + } + + @Override + public Tokens skipWhitespace(MirahLexer l, Input i) { + boolean found_whitespace = false; + ws: + for (int c = i.read(); c != EOF; c = i.read()) { + switch(c) { + case ' ': case '\t': case '\r': case '\f': case 11: + found_whitespace = true; + break; + case '\\': + if (i.consume('\n')) { + l.noteNewline(); + found_whitespace = true; + break; + } + break ws; + case '#': + if (found_whitespace) { + break ws; + } + i.consumeLine(); + return Tokens.tComment; + case '/': + if (i.consume('*')) { + return readBlockComment(l, i); + } + break ws; + default: + break ws; + } + } + i.backup(1); + if (found_whitespace) { + return Tokens.tWhitespace; + } + return null; + } + + private Tokens readBlockComment(MirahLexer l, Input i) { + boolean javadoc = i.peek() == '*'; + for (int c = i.read(); c != EOF; c = i.read()) { + switch(c) { + case '\n': + l.noteNewline(); + break; + case '*': + if (i.consume('/')) { + return javadoc ? Tokens.tJavaDoc : Tokens.tComment; + } + break; + case '/': + if (i.consume('*')) { + readBlockComment(l, i); + } + break; + } + } + return l.unterminatedComment(); + } + + private Tokens processFirstChar(MirahLexer l, Input i) { + Tokens type = null; + int c0 = i.read(); + switch (c0) { + case -1: + if (l.state.hereDocs.isEmpty()) { + type = Tokens.tEOF; + } else { + type = Tokens.tHereDocBegin; + } + break; + case '$': + type = Tokens.tDollar; + break; + case '@': + if (i.consume("@`")) { + type = Tokens.tClassVarBacktick; + } else if (i.consume('@') && i.hasNext()) { + type = Tokens.tClassVar; + } else if (i.consume('`')) { + type = Tokens.tInstVarBacktick; + } else if (i.hasNext()) { + type = Tokens.tInstVar; + } + break; + case '_': + if (i.consume("_ENCODING__")) { + type = Tokens.t__ENCODING__; + } else if (i.consume("_FILE__")) { + type = Tokens.t__FILE__; + } else if (i.consume("_LINE__")) { + type = Tokens.t__LINE__; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'B': + if (i.consume("EGIN")) { + type = Tokens.tBEGIN; + } else { + type = Tokens.tCONSTANT; + } + break; + case 'E': + if (i.consume("ND")) { + type = Tokens.tEND; + } else { + type = Tokens.tCONSTANT; + } + break; + case 'a': + if (i.consume("lias")) { + type = Tokens.tAlias; + } else if (i.consume("nd")) { + type = Tokens.tAnd; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'b': + if (i.consume("egin")) { + type = Tokens.tBegin; + } else if (i.consume("reak")) { + type = Tokens.tBreak; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'c': + if (i.consume("ase")) { + type = Tokens.tCase; + } else if (i.consume("lass")) { + type = Tokens.tClass; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'd': + if (i.consume("efined")) { + type = Tokens.tDefined; + } else if (i.consume("efmacro")) { + type = Tokens.tDefmacro; + } else if (i.consume("ef")) { + type = Tokens.tDef; + } else if (i.consume("o")) { + type = Tokens.tDo; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'e': + if (i.consume("lse")) { + type = Tokens.tElse; + } else if (i.consume("lsif")) { + type = Tokens.tElsif; + } else if (i.consume("nd")) { + type = Tokens.tEnd; + } else if (i.consume("nsure")) { + type = Tokens.tEnsure; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'f': + if (i.consume("alse")) { + type = Tokens.tFalse; + } else if (i.consume("or")) { + type = Tokens.tFor; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'i': + if (i.consume("f")) { + type = Tokens.tIf; + } else if (i.consume("mplements")) { + type = Tokens.tImplements; + } else if (i.consume("mport")) { + type = Tokens.tImport; + } else if (i.consume("nterface")) { + type = Tokens.tInterface; + } else if (i.consume("n")) { + type = Tokens.tIn; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'm': + if (i.consume("acro")) { + type = Tokens.tMacro; + } else if (i.consume("odule")) { + type = Tokens.tModule; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'n': + if (i.consume("ext")) { + type = Tokens.tNext; + } else if (i.consume("il")) { + type = Tokens.tNil; + } else if (i.consume("ot")) { + type = Tokens.tNot; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'o': + if (i.consume("r")) { + type = Tokens.tOr; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'p': + if (i.consume("ackage")) { + type = Tokens.tPackage; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'r': + if (i.consume("aise")) { + type = Tokens.tRaise; + } else if (i.consume("edo")) { + type = Tokens.tRedo; + } else if (i.consume("escue")) { + type = Tokens.tRescue; + } else if (i.consume("etry")) { + type = Tokens.tRetry; + } else if (i.consume("eturn")) { + type = Tokens.tReturn; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 's': + if (i.consume("elf")) { + type = Tokens.tSelf; + } else if (i.consume("uper")) { + type = Tokens.tSuper; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 't': + if (i.consume("hen")) { + type = Tokens.tThen; + } else if (i.consume("rue")) { + type = Tokens.tTrue; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'u': + if (i.consume("ndef")) { + type = Tokens.tUndef; + } else if (i.consume("nless")) { + type = Tokens.tUnless; + } else if (i.consume("ntil")) { + type = Tokens.tUntil; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'w': + if (i.consume("hen")) { + type = Tokens.tWhen; + } else if (i.consume("hile")) { + type = Tokens.tWhile; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case 'y': + if (i.consume("ield")) { + type = Tokens.tYield; + } else { + type = Tokens.tIDENTIFIER; + } + break; + case '\n': + l.noteNewline(); + if (l.state.hereDocs.isEmpty()) { + type = Tokens.tNL; + } else { + type = Tokens.tHereDocBegin; + l.pushState(l.state.hereDocs.removeFirst()); + } + break; + case '/': + if (l.isBEG()) { + l.pushState(new RegexLexer()); + type = Tokens.tRegexBegin; + } else if (!i.hasNext()) { + type = Tokens.tSlash; + } else if (i.consume('=')) { + type = Tokens.tOpAssign; + } else if (l.isARG() && l.spaceSeen && !Character.isWhitespace(i.peek())) { + // warn("Ambiguous first argument; make sure.") + type = Tokens.tRegexBegin; + } else { + type = Tokens.tSlash; + } + break; + case '\'': + l.pushState(new SStringLexer()); + type = Tokens.tSQuote; + break; + case '"': + l.pushState(new DStringLexer()); + type = Tokens.tDQuote; + break; + case ':': + if (i.consume(':')) { + type = Tokens.tColons; + } else { + type = Tokens.tColon; + } + break; + case '.': + if (i.consume('.')) { + type = Tokens.tDots; + } else { + type = Tokens.tDot; + } + break; + case '(': + type = Tokens.tLParen; + break; + case ')': + type = Tokens.tRParen; + break; + case '[': + type = Tokens.tLBrack; + break; + case ']': + type = Tokens.tRBrack; + break; + case '{': + l.state.lbrace(); + type = Tokens.tLBrace; + break; + case '}': + l.state.rbrace(l); + type = Tokens.tRBrace; + break; + case ';': + type = Tokens.tSemi; + break; + case '!': + if (i.consume('=')) { + type = Tokens.tNE; + } else if (i.consume('~')) { + type = Tokens.tNMatch; + } else { + type = Tokens.tBang; + } + break; + case '<': + if (i.consume('=')) { + if (i.consume('>')) { + type = Tokens.tLEG; + } else { + type = Tokens.tLE; + } + } else if (i.consume('<')) { + if (i.consume('<')) { + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tLLShift; + } + } else if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tLShift; + } + } else { + type = Tokens.tLT; + } + break; + case '>': + if (i.consume('=')) { + type = Tokens.tGE; + } else if (i.consume('>')) { + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tRShift; + } + } else { + type = Tokens.tGT; + } + break; + case '?': + type = Tokens.tQuestion; + break; + case '=': + if (i.consume('>')) { + type = Tokens.tRocket; + } else if (i.consume('~')) { + type = Tokens.tMatch; + } else if (i.consume('=')) { + if (i.consume('=')) { + type = Tokens.tEEEQ; + } else { + type = Tokens.tEEQ; + } + } else { + type = Tokens.tEQ; + } + break; + case '&': + if (i.consume('&')) { + if (i.consume('=')) { + type = Tokens.tAndEq; + } else { + type = Tokens.tAmpers; + } + } else if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tAmper; + } + break; + case '|': + if (i.consume('|')) { + if (i.consume('=')) { + type = Tokens.tOrEq; + } else { + type = Tokens.tPipes; + } + } else if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tPipe; + } + break; + case '*': + if (i.consume('*')) { + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tStars; + } + } else if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tStar; + } + break; + case '+': + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tPlus; + } + break; + case '-': + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tMinus; + } + break; + case '%': + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tPercent; + } + break; + case '^': + if (i.consume('=')) { + type = Tokens.tOpAssign; + } else { + type = Tokens.tCaret; + } + break; + case '~': + type = Tokens.tTilde; + break; + case '`': + type = Tokens.tBacktick; + break; + case ',': + type = Tokens.tComma; + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + type = Tokens.tDigit; + break; + default: + // Note, I'm treating all surrogate pairs as identifier chars. + // We can fix that if it's ever a problem. + if ((c0 >= 'A' && c0 <= 'Z') || + Character.isUpperCase(c0)) { + type = Tokens.tCONSTANT; + } else if (c0 >= 0xD800 && c0 <= 0xDBFF) { + if (Character.isUpperCase(i.finishCodepoint())) { + type = Tokens.tCONSTANT; + } else { + type = Tokens.tIDENTIFIER; + } + } else if ((c0 >= 'a' && c0 <= 'z') || + (c0 > 0x7f && Character.isLetter(c0))) { + type = Tokens.tIDENTIFIER; + } else { + type = Tokens.tUNKNOWN; + } + } + return type; + } + + private Tokens checkKeyword(Tokens type, Input i) { + // If we found a keyword, see if it's just the beginning of another name + if (Tokens.tClassVar.compareTo(type) > 0) { + int c1 = i.peek(); + if ((c1 >= 0xD800 && c1 <= 0xDFFF) || + Character.isLetterOrDigit(c1) || + c1 == '_') { + if (type == Tokens.tBEGIN || type == Tokens.tEND) { + type = Tokens.tCONSTANT; + } else { + type = Tokens.tIDENTIFIER; + } + } + } + return type; + } + + private Tokens readRestOfToken(Tokens type, MirahLexer l, Input i) { + switch (type) { + case tDigit: + return readNumber(i); + case tQuestion: + return readCharacter(i); + case tLShift: + return readHereDocIdentifier(l, i); + } + if (Tokens.tFID.compareTo(type) >= 0) { + return readName(i, type); + } + return type; + } + + private Tokens readName(Input i, Tokens type) { + int start = i.pos(); + for (int c = i.read(); c != EOF; c = i.read()) { + if (c <= 0x7F) { + if (c == '_' || + (c >= '0' && c <= '9') || + (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z')) { + continue; + } else if (c == '?' || c == '!') { + if (i.consume('=')) { + i.backup(1); + break; + } else { + return Tokens.tFID; + } + } + break; + } else if (c >= 0xD800 && c <= 0xDFFF) { + continue; + } else { + if (!Character.isLetterOrDigit(c)) { + break; + } + } + } + i.backup(1); + if (type == Tokens.tInstVar && i.pos() == start) { + type = Tokens.tAt; + } + return type; + } + + private void readDigits(Input i, boolean oct, boolean dec, boolean hex) { + loop: + for (int c = i.read(); c != EOF; c = i.read()) { + switch (c) { + case '0': case '1': case '_': + break; + case '2': case '3': case '4': case '5': case '6': case '7': + if (oct) { + break; + } else { + i.backup(1); + return; + } + case '8': case '9': + if (dec) { + break; + } else { + i.backup(1); + return; + } + case 'a': case 'A': case 'b': case 'B': case 'c': case 'C': + case 'd': case 'D': case 'e': case 'E': case 'f': case 'F': + if (hex) { + break; + } else { + i.backup(1); + return; + } + default: + i.backup(1); + return; + } + } + i.backup(1); + } + + private Tokens readNumber(Input i) { + if (!i.hasNext()) { + return Tokens.tInteger; + } + boolean oct = true; + boolean dec = true; + boolean hex = false; + boolean maybeFloat = true; + i.backup(1); + if (i.consume('0')) { + switch (i.read()) { + case 'd': case 'D': + maybeFloat = false; + break; + case 'b': case 'B': + oct = dec = maybeFloat = false; + break; + case 'x': case 'X': + hex = true; + maybeFloat = false; + break; + case '.': case 'e': case 'E': + i.backup(1); + break; + case 'o': case 'O': + dec = false; + break; + default: + i.backup(1); + maybeFloat = false; + dec = false; + } + } + readDigits(i, oct, dec, hex); + if (maybeFloat && i.hasNext()) { + if (i.consume('.')) { + if (readFraction(i)) { + return Tokens.tFloat; + } else { + i.backup(1); + } + } else if (i.consume('e') || i.consume('E')) { + readExponent(i); + return Tokens.tFloat; + } + } + return Tokens.tInteger; + } + + private boolean readFraction(Input i) { + int start = i.pos(); + readDigits(i, true, true, false); + if (start == i.pos()) { + return false; + } + if (i.hasNext()) { + if (i.consume('e') || i.consume('E')) { + readExponent(i); + return true; + } + } + return true; + } + + private void readExponent(Input i) { + i.consume('-'); + readDigits(i, true, true, false); + } + + private boolean isIdentifierChar(int c) { + if (c == EOF) { + return false; + } + return Character.isLetterOrDigit(c) || c == '_' || (c >= 0xD800 && c <= 0xDBFF); + } + + private Tokens readCharacter(Input i) { + if (!i.hasNext()) { + return Tokens.tQuestion; + } + int c = i.read(); + if (c == EOF || Character.isWhitespace(c)) { + i.backup(1); + return Tokens.tQuestion; + } + if ((c == '_' || Character.isLetterOrDigit(c)) && + isIdentifierChar(i.peek())) { + i.backup(1); + return Tokens.tQuestion; + } + if (c >= 0xD800 && c <= 0xDBFF) { + i.skip(1); + if (isIdentifierChar(i.peek())) { + i.backup(2); + return Tokens.tQuestion; + } + return Tokens.tCharacter; + } + if (c != '\\') { + return Tokens.tCharacter; + } +// if (i.consume('\\')) { +// return Tokens.tCharacter; +// } + // Just gobble up any digits. Let the parser worry about whether it's a + // valid escape. + while (EOF != (c = i.read())) { + switch (c) { + case '0': case '1': case '2': case '3': case '4': case '5': case '6': + case '7': case '8': case '9': case 'a': case 'A': case 'b': case 'B': + case 'c': case 'C': case 'd': case 'D': case 'e': case 'E': case 'f': + case 'F': case 'x': case 'u': case 'U': + continue; + } + break; + } + if (c == EOF || Character.isWhitespace(c)) { + i.backup(1); + } + return Tokens.tCharacter; + } + + private Tokens readHereDocIdentifier(MirahLexer l, Input i) { + int start = i.pos(); + boolean allowIndented = false; + if (!i.hasNext() || l.isEND() || (l.isARG() && !l.spaceSeen)) { + return Tokens.tLShift; + } + if (i.consume('-')) { + allowIndented = true; + } + char quote = 0; + if (i.consume('"')) { + quote = '"'; + } else if (i.consume('\'')) { + quote = '\''; + } + int id_start = i.pos(); + for (int c = i.read(); c != EOF; c = i.read()) { + if (!isIdentifierChar(c)) { + break; + } + } + i.backup(1); + if (i.pos() == id_start) { + return Tokens.tLShift; + } + CharSequence id = i.readBack(i.pos() - id_start); + if (quote != 0) { + if (!i.consume(quote)) { + i.backup(i.pos() - start); + return Tokens.tLShift; + } + } + l.state.hereDocs.add(new HereDocLexer(id.toString(), allowIndented, quote != '\'')); + return Tokens.tHereDocId; + } + } + + public MirahLexer(String string, char[] chars, BaseParser parser) { + this(new StringInput(string, chars)); + this.parser = parser; + } + + public MirahLexer(Input input) { + this.input = input; + pushState(new StandardLexer()); + argTokens = EnumSet.of(Tokens.tSuper, Tokens.tYield, Tokens.tIDENTIFIER, + Tokens.tCONSTANT, Tokens.tFID); + beginTokens = EnumSet.range(Tokens.tBang, Tokens.tOpAssign); + beginTokens.addAll(EnumSet.of( + Tokens.tElse, Tokens.tCase, Tokens.tEnsure, /*Tokens.tModule,*/ + Tokens.tElsif, Tokens.tNot, Tokens.tThen, Tokens.tFor, Tokens.tReturn, + Tokens.tIf, Tokens.tIn, Tokens.tDo, Tokens.tUntil, Tokens.tUnless, + Tokens.tOr, Tokens.tWhen, Tokens.tAnd, Tokens.tBegin, Tokens.tWhile, + Tokens.tNL, Tokens.tSemi, Tokens.tColon, Tokens.tSlash, Tokens.tLBrace, + Tokens.tLBrack, Tokens.tLParen, Tokens.tDots)); + beginTokens.addAll(EnumSet.range(Tokens.tComma, Tokens.tRocket)); + // Comment? + endTokens = EnumSet.of( + Tokens.tDot, Tokens.tCharacter, Tokens.tSQuote, Tokens.tDQuote, + Tokens.tRParen, Tokens.tRBrace, Tokens.tRBrack, Tokens.tRegexEnd, + Tokens.tInteger, Tokens.tFloat, Tokens.tInstVar, Tokens.tClassVar, + Tokens.tEnd, Tokens.tSelf, Tokens.tFalse, Tokens.tTrue, Tokens.tRetry, + Tokens.tBreak, Tokens.tNil, Tokens.tNext, Tokens.tRedo, Tokens.tClass, + Tokens.tDef); + } + + private boolean isBEG() { + return isBEG; + } + + private boolean isARG() { + return isARG; + } + + private boolean isEND() { + return isEND; + } + + private void pushState(Lexer lexer) { + this.state = new State(this.state, lexer); + } + + private void pushForOneToken(Lexer lexer) { + this.state = new State(this.state, lexer, true); + } + + private void popState() { + if (state.previous != null) { + state = state.previous; + } + } + + public Tokens simpleLex() { + boolean shouldPop = state.justOnce; + Tokens type = state.lexer.skipWhitespace(this, input); + if (type != null) { + spaceSeen = true; + return type; + } + type = state.lexer.lex(this, input); + if (shouldPop) { + popState(); + } + spaceSeen = false; + isBEG = beginTokens.contains(type); + isARG = argTokens.contains(type); + isEND = endTokens.contains(type); + return type; + } + + public Token lex(int pos) { + if (pos < input.pos()) { + ListIterator> it = tokens.listIterator(tokens.size()); + while (it.hasPrevious()) { + Token savedToken = it.previous(); + if (pos >= savedToken.pos && pos <= savedToken.startpos) { + logger.fine("Warning, uncached token " + savedToken.type + " at " + pos); + parser._pos = savedToken.endpos; + return savedToken; + } + } + throw new IllegalArgumentException("" + pos + " < " + input.pos()); + } else if (!input.hasNext()) { + return parser.build_token(state.hereDocs.isEmpty() ? Tokens.tEOF : Tokens.tHereDocBegin, pos, pos); + } + Tokens type = Tokens.tWhitespace; + int start = input.pos(); + while (type.ordinal() > Tokens.tEOF.ordinal()) { + start = input.pos(); + type = simpleLex(); + } + parser._pos = input.pos(); + Token token = parser.build_token(type, pos, start); + tokens.add(token); + return token; + } + + void noteNewline() { + if (parser != null) { + parser.note_newline(input.pos()); + } + } + + public Tokens unterminatedComment() { + if (parser == null) { + return Tokens.tPartialComment; + } else { + throw parser.syntaxError("*/", null); + } + } + + public Object getState() { + if (state.previous == null && state.hereDocs.isEmpty() && state.braceDepth < 0x20) { + int compressed = 0; + if (isBEG) { + compressed = 1; + } else if (isEND) { + compressed = 2; + } else if (isARG) { + if (spaceSeen) { + compressed = 4; + } else { + compressed = 3; + } + } + compressed |= (state.braceDepth << 3); + return compressed; + } + return new CombinedState(this); + } + + public void restore(Object state) { + if (state instanceof CombinedState) { + CombinedState cs = (CombinedState)state; + this.state = cs.state; + spaceSeen = cs.spaceSeen; + isBEG = cs.isBEG; + isARG = cs.isARG; + isEND = cs.isEND; + } else { + int compressed = ((Integer)state).intValue(); + this.state = new State(null, new StandardLexer()); + this.state.braceDepth = (compressed >> 3) & 0x1f; + isBEG = isARG = isEND = spaceSeen = false; + switch (compressed & 0x7) { + case 1: + isBEG = true; + break; + case 2: + isEND = true; + break; + case 4: + spaceSeen = true; + // fall through; + case 3: + isARG = true; + break; + } + } + } + + private Input input; + private BaseParser parser; + private State state; + private ArrayList> tokens = new ArrayList>(); + private EnumSet beginTokens; + private EnumSet argTokens; + private EnumSet endTokens; + private boolean spaceSeen = false; + private boolean isBEG = true; + private boolean isARG = false; + private boolean isEND = false; +} diff --git a/src/mirah/lang/ast/ast.mirah b/src/mirah/lang/ast/ast.mirah index 9339ec5..ba012dc 100644 --- a/src/mirah/lang/ast/ast.mirah +++ b/src/mirah/lang/ast/ast.mirah @@ -83,6 +83,10 @@ interface Annotated < Node do # end end +interface HasModifiers < Node do + def modifiers:ModifierList;end +end + # Should this go somewhere else? # Should this support multi-dimensional arrays? interface TypeRef < TypeName do diff --git a/src/mirah/lang/ast/intrinsics.mirah b/src/mirah/lang/ast/intrinsics.mirah index b7c543a..36c6a1f 100644 --- a/src/mirah/lang/ast/intrinsics.mirah +++ b/src/mirah/lang/ast/intrinsics.mirah @@ -170,5 +170,14 @@ class MacroDefinition < NodeImpl child_list body: Node child_list annotations: Annotation attr_accessor isStatic: 'boolean' + child java_doc: Node + end + + def initialize(name: Identifier, arguments: Arguments, body: List, annotations: List) + initialize(name, arguments, body, annotations, Node(nil)) + end + + def initialize(p:Position, name: Identifier, arguments: Arguments, body: List, annotations: List) + initialize(p, name, arguments, body, annotations, Node(nil)) end end \ No newline at end of file diff --git a/src/mirah/lang/ast/klass.mirah b/src/mirah/lang/ast/klass.mirah index 2c6b9f5..19658a2 100644 --- a/src/mirah/lang/ast/klass.mirah +++ b/src/mirah/lang/ast/klass.mirah @@ -4,48 +4,84 @@ import java.util.List # Note: com.sun.source.tree uses the same node for classes, interfaces, annotations, etc. class ClassDefinition < NodeImpl - implements Annotated, Named + implements Annotated, Named, HasModifiers init_node do child name: Identifier child superclass: TypeName child_list body: Node child_list interfaces: TypeName child_list annotations: Annotation + child_list modifiers: Modifier + child java_doc: Node end + + def initialize(p:Position, name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(p, name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end + + def initialize(name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end + end class InterfaceDeclaration < ClassDefinition init_subclass(ClassDefinition) + + def initialize(p:Position, name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(p, name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end + + def initialize(name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end end # Is this necessary? class ClosureDefinition < ClassDefinition init_subclass(ClassDefinition) + + def initialize(p:Position, name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(p, name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end + + def initialize(name:Identifier, super_class: TypeName, body: List, interfaces: List, annotations: List, modifiers: List) + initialize(name, super_class, body, interfaces, annotations, modifiers, Node(nil)) + end end class FieldDeclaration < NodeImpl - implements Annotated, Named + implements Annotated, Named, HasModifiers init_node do child name: Identifier child type: TypeName child_list annotations: Annotation attr_accessor isStatic: 'boolean' + child_list modifiers: Modifier end end class FieldAssign < NodeImpl - implements Annotated, Named, Assignment + implements Annotated, Named, Assignment, HasModifiers init_node do child name: Identifier child value: Node child_list annotations: Annotation attr_accessor isStatic: 'boolean' + child_list modifiers: Modifier + end + + def initialize(position:Position, name:Identifier, annotations:List, isStatic:boolean, modifiers: List) + initialize(position, name, Node(nil), annotations, modifiers) + self.isStatic = isStatic end def initialize(position:Position, name:Identifier, annotations:List, isStatic:boolean) - initialize(position, name, Node(nil), annotations) + initialize(position, name, Node(nil), annotations, nil) self.isStatic = isStatic end + + end class FieldAccess < NodeImpl @@ -90,11 +126,12 @@ class Colon3 < Constant end class ConstantAssign < NodeImpl - implements Annotated, Named, Assignment + implements Annotated, Named, Assignment, HasModifiers init_node do child name: Identifier child value: Node child_list annotations: Annotation + child_list modifiers: Modifier end end diff --git a/src/mirah/lang/ast/lists.mirah b/src/mirah/lang/ast/lists.mirah index 399e156..fb3d4df 100644 --- a/src/mirah/lang/ast/lists.mirah +++ b/src/mirah/lang/ast/lists.mirah @@ -31,3 +31,7 @@ end class OptionalArgumentList < NodeImpl init_list OptionalArgument end + +class ModifierList < NodeImpl + init_list Modifier +end diff --git a/src/mirah/lang/ast/method.mirah b/src/mirah/lang/ast/method.mirah index 96e777b..84196e7 100644 --- a/src/mirah/lang/ast/method.mirah +++ b/src/mirah/lang/ast/method.mirah @@ -16,6 +16,7 @@ package mirahparser.lang.ast import java.util.Collections +import java.util.List interface FormalArgument < Named do def name:Identifier; end @@ -77,21 +78,47 @@ class BlockArgument < NodeImpl end class MethodDefinition < NodeImpl - implements Named, Annotated + implements Named, Annotated, HasModifiers init_node do child name: Identifier child arguments: Arguments child type: TypeName child_list body: Node child_list annotations: Annotation - # exceptions + child_list modifiers: Modifier + child java_doc: Node end + + def initialize(p:Position, name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(p, name, arguments, type, body, annotations, modifiers, Node(nil)) + end + + def initialize(name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(name, arguments, type, body, annotations, modifiers, Node(nil)) + end + end class StaticMethodDefinition < MethodDefinition init_subclass(MethodDefinition) + + def initialize(p:Position, name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(p, name, arguments, type, body, annotations, modifiers, Node(nil)) + end + + def initialize(name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(name, arguments, type, body, annotations, modifiers, Node(nil)) + end end class ConstructorDefinition < MethodDefinition init_subclass(MethodDefinition) + + def initialize(p:Position, name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(p, name, arguments, type, body, annotations, modifiers, Node(nil)) + end + + def initialize(name: Identifier, arguments: Arguments, type:TypeName, body: List, annotations: List, modifiers:List) + initialize(name, arguments, type, body, annotations, modifiers, Node(nil)) + end end \ No newline at end of file diff --git a/src/mirah/lang/ast/structure.mirah b/src/mirah/lang/ast/structure.mirah index e77d3e0..b8c67be 100644 --- a/src/mirah/lang/ast/structure.mirah +++ b/src/mirah/lang/ast/structure.mirah @@ -48,9 +48,18 @@ class Script < NodeImpl end end +class JavaDoc < NodeImpl + init_literal String +end + class Annotation < NodeImpl init_node do child type: TypeName child_list values: HashEntry end +end + + +class Modifier < NodeImpl + init_literal String end \ No newline at end of file diff --git a/src/mirahparser/impl/Mirah.mmeta b/src/mirahparser/impl/Mirah.mmeta index 9b8f714..b27fd7f 100644 --- a/src/mirahparser/impl/Mirah.mmeta +++ b/src/mirahparser/impl/Mirah.mmeta @@ -212,6 +212,8 @@ parser MirahParser { | ($Def)=> primary2_def | ($Macro|$Defmacro)=> primary2_macro | ($Dollar)=> (primary2_class | primary2_interface | primary2_def | primary2_macro) + | (java_doc) => (primary2_class | primary2_interface | primary2_def | primary2_macro) + | (acc_modifier)=> (primary2_class | primary2_interface | primary2_def | primary2_macro) | ($Break)=> primary2_break | ($Next)=> primary2_next | ($Redo)=> primary2_redo @@ -298,7 +300,9 @@ parser MirahParser { $Scope[@BEG] $Returns[Node] primary2_class - : annotations=annotation_list + : jdoc = java_doc? + annotations=annotation_list + m=acc_modifier_list t=$Class sp=block_start(t)! ( $LShift! BEG e=$Self term! b=compstmt! verify_end(sp)! $End @@ -309,12 +313,13 @@ parser MirahParser { # sense for mirah. Maybe rescuing exceptions # in the static initializer? verify_end(sp)! $End - -> ^(ClassDefinition n s b ifaces annotations) + -> ^(ClassDefinition n s b ifaces annotations m jdoc) ); $Scope[@cond] $Returns[Node] primary2_interface - : annotations=annotation_list + : jdoc = java_doc? + annotations=annotation_list t=$Interface! sp=block_start(t) n=cname! ifaces=($LT! rlistOf(:cpath) @@ -323,7 +328,7 @@ parser MirahParser { Do! b=compstmt! verify_end(sp) $End - -> ^(InterfaceDeclaration n nil b ifaces annotations) + -> ^(InterfaceDeclaration n nil b ifaces annotations nil jdoc) ; $Returns[Node] primary2_import @@ -360,30 +365,33 @@ parser MirahParser { $Scope[@BEG] $Returns[Node] primary2_def - : annotations=annotation_list + : jdoc = java_doc? + annotations=annotation_list + m=acc_modifier_list t=$Def s=block_start(t)! ( $Self! dot_or_colon! name=fname! arglist=f_arglist! body=bodystmt! verify_end(s)! $End {args = arglist.args; type=arglist.returnType} - -> ^(StaticMethodDefinition name args type body annotations) + -> ^(StaticMethodDefinition name args type body annotations m jdoc) | name=fname! arglist=f_arglist! body=bodystmt! verify_end(s)! $End {args = arglist.args; type=arglist.returnType} ( ?{nameIs(name, "initialize")} - -> ^(ConstructorDefinition name args type body annotations) - | -> ^(MethodDefinition name args type body annotations) + -> ^(ConstructorDefinition name args type body annotations m jdoc) + | -> ^(MethodDefinition name args type body annotations m jdoc) ) ); $Scope[@BEG] $Returns[Node] primary2_macro - : annotations=annotation_list + : jdoc = java_doc? + annotations=annotation_list (t=$Defmacro! | t=$Macro! $Def) s=block_start(t) isStatic=($Self dot_or_colon)? name=fname! ($LParen! opt_nl args=f_args opt_nl $RParen)? (?{t == Tokens.tDefmacro} $Do)? body=bodystmt! verify_end(s)! $End - node=-> ^(MacroDefinition name args body annotations) + node=-> ^(MacroDefinition name args body annotations jdoc) {node.isStatic = (isStatic != nil); node} ; primary2_break: $Break -> ^(Break); @@ -637,7 +645,7 @@ parser MirahParser { $Memo[Assignment] lhs: (($ClassVar|$InstVar|$CONSTANT|$IDENTIFIER) $EQ)=> var_lhs - | ($Dollar|$Backtick|$ClassVarBacktick|$InstVarBacktick)=> (var_lhs | lhs2) + | ($Dollar|$Backtick|$ClassVarBacktick|$InstVarBacktick|acc_modifier)=> (var_lhs | lhs2) # | ($Colons $CONSTANT)=> colon3 n=constant -> ^(ConstantAssign ^(Colon3 n)) | lhs2; $Returns[Assignment] @@ -651,9 +659,10 @@ parser MirahParser { $Returns[Assignment] var_lhs: ( annotations=annotation_list - ( ($ClassVar|$ClassVarBacktick)=> n=cvar -> ^(FieldAssign n annotations true) - | ($InstVar|$InstVarBacktick)=> n=ivar -> ^(FieldAssign n annotations false) - | ($CONSTANT)=> n=constant -> ^(ConstantAssign n nil annotations) + m=acc_modifier_list + ( ($ClassVar|$ClassVarBacktick)=> n=cvar -> ^(FieldAssign n annotations true m) + | ($InstVar|$InstVarBacktick)=> n=ivar -> ^(FieldAssign n annotations false m) + | ($CONSTANT)=> n=constant -> ^(ConstantAssign n nil annotations m) ) ) | ($IDENTIFIER)=> n=identifier -> ^(LocalAssignment n nil) @@ -701,6 +710,26 @@ parser MirahParser { $Returns[List] annotation_list: (a=annotation opt_nl {a})*; + $Returns[Node] + acc_modifier: txt = ($ACC_ABSTRACT {"ABSTRACT"}| + $ACC_PUBLIC {"PUBLIC"} | + $ACC_PRIVATE {"PRIVATE"} | + $ACC_PROTECTED {"PROTECTED"} | + $ACC_FINAL {"FINAL"} | + $ACC_VOLATILE {"VOLATILE"} | + $ACC_NATIVE {"NATIVE"} | + $ACC_DEFAULT {"DEFAULT"} | + $ACC_SYNCHRONIZED {"SYNCHRONIZED"} | + $ACC_TRANSIENT {"TRANSIENT"} + ) + -> ^(Modifier txt); + + $Returns[List] + acc_modifier_list: (a=acc_modifier {a})*; + + $Returns[Node] + java_doc: txt = (s = $JavaDoc opt_nl {Token(s).text}) -> ^(JavaDoc txt); + op: text=($LT {"<"} | $LE {"<="} | $LEG {"<=>"} @@ -960,7 +989,7 @@ parser MirahParser { } $Memo[Token] - _lex: {@lexer.lex(_pos)}; + _lex: {@lexer.lex(_pos, @skipWhitespaceAndComments, @skipJavaDoc)}; def _lex(type:Tokens) { token = _lex @@ -1043,6 +1072,16 @@ parser MirahParser { @reserved = EnumSet.range(Tokens.tBEGIN, Tokens.tYield) @heredocs = LinkedList.new @escape = EscapeParser.new + @skipWhitespaceAndComments = true + @skipJavaDoc = true + } + + def skip_whitespace(skip:boolean=true) { + @skipWhitespaceAndComments = skip + } + + def skip_java_doc(skip:boolean=true) { + @skipJavaDoc = skip } def init { diff --git a/src/mirahparser/impl/MirahLexer.java b/src/mirahparser/impl/MirahLexer.java index 9431369..9d8ad1a 100644 --- a/src/mirahparser/impl/MirahLexer.java +++ b/src/mirahparser/impl/MirahLexer.java @@ -435,6 +435,9 @@ public Tokens skipWhitespace(MirahLexer l, Input i) { i.consumeLine(); return Tokens.tComment; case '/': + if (found_whitespace) { + break ws; + } if (i.consume('*')) { return readBlockComment(l, i); } @@ -523,7 +526,9 @@ private Tokens processFirstChar(MirahLexer l, Input i) { } break; case 'a': - if (i.consume("lias")) { + if (i.consume("bstract")) { + type = Tokens.tACC_ABSTRACT; + } else if (i.consume("lias")) { type = Tokens.tAlias; } else if (i.consume("nd")) { type = Tokens.tAnd; @@ -552,6 +557,8 @@ private Tokens processFirstChar(MirahLexer l, Input i) { case 'd': if (i.consume("efined")) { type = Tokens.tDefined; + } else if (i.consume("efault")) { + type = Tokens.tACC_DEFAULT; } else if (i.consume("efmacro")) { type = Tokens.tDefmacro; } else if (i.consume("ef")) { @@ -578,6 +585,8 @@ private Tokens processFirstChar(MirahLexer l, Input i) { case 'f': if (i.consume("alse")) { type = Tokens.tFalse; + } else if (i.consume("inal")) { + type = Tokens.tACC_FINAL; } else if (i.consume("or")) { type = Tokens.tFor; } else { @@ -611,6 +620,8 @@ private Tokens processFirstChar(MirahLexer l, Input i) { case 'n': if (i.consume("ext")) { type = Tokens.tNext; + } else if (i.consume("ative")) { + type = Tokens.tACC_NATIVE; } else if (i.consume("il")) { type = Tokens.tNil; } else if (i.consume("ot")) { @@ -629,6 +640,12 @@ private Tokens processFirstChar(MirahLexer l, Input i) { case 'p': if (i.consume("ackage")) { type = Tokens.tPackage; + } else if (i.consume("rivate")) { + type = Tokens.tACC_PRIVATE; + } else if (i.consume("rotected")) { + type = Tokens.tACC_PROTECTED; + }else if (i.consume("ublic")) { + type = Tokens.tACC_PUBLIC; } else { type = Tokens.tIDENTIFIER; } @@ -653,6 +670,8 @@ private Tokens processFirstChar(MirahLexer l, Input i) { type = Tokens.tSelf; } else if (i.consume("uper")) { type = Tokens.tSuper; + } else if (i.consume("ynchronized")) { + type = Tokens.tACC_SYNCHRONIZED; } else { type = Tokens.tIDENTIFIER; } @@ -660,7 +679,9 @@ private Tokens processFirstChar(MirahLexer l, Input i) { case 't': if (i.consume("hen")) { type = Tokens.tThen; - } else if (i.consume("rue")) { + } else if (i.consume("ransient")) { + type = Tokens.tACC_TRANSIENT; + }else if (i.consume("rue")) { type = Tokens.tTrue; } else { type = Tokens.tIDENTIFIER; @@ -677,6 +698,13 @@ private Tokens processFirstChar(MirahLexer l, Input i) { type = Tokens.tIDENTIFIER; } break; + case 'v': + if (i.consume("olatile")) { + type = Tokens.tACC_VOLATILE; + } else { + type = Tokens.tIDENTIFIER; + } + break; case 'w': if (i.consume("hen")) { type = Tokens.tWhen; @@ -1266,39 +1294,47 @@ public Tokens simpleLex() { public Token lex(int pos) { - return lex(pos, true); + return lex(pos, true, true); } - public Token lex(int pos, boolean skipWhitespaceAndComments) { - if (pos < input.pos()) { - ListIterator> it = tokens.listIterator(tokens.size()); - while (it.hasPrevious()) { - Token savedToken = it.previous(); - if (pos >= savedToken.pos && pos <= savedToken.startpos) { - logger.fine("Warning, uncached token " + savedToken.type + " at " + pos); - parser._pos = savedToken.endpos; - return savedToken; - } + public Token lex(int pos, boolean skipWhitespaceAndAllComments) { + return lex(pos, skipWhitespaceAndAllComments, skipWhitespaceAndAllComments); + } + + public Token lex(int pos, boolean skipWhitespaceAndComments, boolean skipJavaDocs) { + if (pos < input.pos()) { + ListIterator> it = tokens.listIterator(tokens.size()); + while (it.hasPrevious()) { + Token savedToken = it.previous(); + if (pos >= savedToken.pos && pos <= savedToken.startpos) { + logger.fine("Warning, uncached token " + savedToken.type + " at " + pos); + parser._pos = savedToken.endpos; + return savedToken; + } + } + throw new IllegalArgumentException("" + pos + " < " + input.pos()); + } else if (!input.hasNext()) { + return parser.build_token(state.hereDocs.isEmpty() ? Tokens.tEOF : Tokens.tHereDocBegin, pos, pos); } - throw new IllegalArgumentException("" + pos + " < " + input.pos()); - } else if (!input.hasNext()) { - return parser.build_token(state.hereDocs.isEmpty() ? Tokens.tEOF : Tokens.tHereDocBegin, pos, pos); - } - Tokens type = Tokens.tWhitespace; - int start = input.pos(); - if (skipWhitespaceAndComments) { - while (type.ordinal() > Tokens.tEOF.ordinal()) { + Tokens type; + int start; + + while (true) { start = input.pos(); type = simpleLex(); - } - } else { - start = input.pos(); - type = simpleLex(); - } - parser._pos = input.pos(); - Token token = parser.build_token(type, pos, start); - tokens.add(token); - return token; + if (skipWhitespaceAndComments && (type == Tokens.tWhitespace || type == Tokens.tComment)) { + continue; + } else if (skipJavaDocs && type == Tokens.tJavaDoc) { + continue; + } else { + break; + } + } + + parser._pos = input.pos(); + Token token = parser.build_token(type, pos, start); + tokens.add(token); + return token; } void noteNewline() { diff --git a/src/mirahparser/impl/Tokens.java b/src/mirahparser/impl/Tokens.java index f81c6ea..39ef16d 100644 --- a/src/mirahparser/impl/Tokens.java +++ b/src/mirahparser/impl/Tokens.java @@ -66,7 +66,17 @@ public enum Tokens { tWhen, tWhile, tYield, - // ClassVar must be first after the keywords, followed by other + tACC_ABSTRACT, + tACC_PUBLIC, + tACC_PRIVATE, + tACC_PROTECTED, + tACC_FINAL, + tACC_VOLATILE, + tACC_NATIVE, + tACC_DEFAULT, + tACC_TRANSIENT, + tACC_SYNCHRONIZED, + // ClassVar must be first after the keywords, followed by other // identifierish tokens. tClassVar, tInstVar, diff --git a/test/test_lexer.rb b/test/test_lexer.rb new file mode 100644 index 0000000..2088dc4 --- /dev/null +++ b/test/test_lexer.rb @@ -0,0 +1,54 @@ +require 'test/unit' +require 'java' + +$CLASSPATH << 'build/mirah-parser.jar' + +class TestParsing < Test::Unit::TestCase + java_import 'mirahparser.impl.MirahLexer' + java_import 'mirahparser.lang.ast.StringCodeSource' + java_import 'mirahparser.impl.MirahParser' + java_import 'mirahparser.impl.Tokens' + java_import 'org.mirahparser.mmeta.SyntaxError' + + def parse(text, ignore_spaces, ignore_comments) + parser = MirahParser.new + parser.init(text) + lexer = MirahLexer.new parser._string, parser._chars , parser + result = [] + pos = 0 + while (token = lexer.lex(pos, ignore_spaces, ignore_comments)).type != Tokens::tEOF + result << token + pos = token.endpos + end + result.join(",") + end + + def assert_parse(expected, text , ignore_spaces, ignore_comments) + tokens = parse(text, ignore_spaces, ignore_comments) + assert_equal(expected, tokens, "expected '#{text}' to be converted ignore spaces: #{ignore_spaces}, #{ignore_comments} " ) + end + + def assert_fails(text) + begin + fail("Should raise syntax error, but got #{parse text, true, true}") + rescue SyntaxError + # ok + end + end + + def test_fixnum + assert_parse(',,,', 'class X; end', true, true) + assert_parse('', ' #class X; end', true, true) + assert_parse('', ' /** jdoc */ ', true, true) + assert_fails('/*not finished ') + + assert_parse(%q{,,,,,}, 'class X; end', false, false) + assert_parse(%q{,}, ' #class X; end ', false, false) + assert_parse(%q{,,}, ' /** jdoc */ ', false ,false) + + assert_parse(%q{,,,}, 'class X; end', true, false) + assert_parse('', ' #class X; end ', true, false) + assert_parse(%q{}, ' /** jdoc */ ', true, false) + + end +end diff --git a/test/test_mirah.rb b/test/test_mirah.rb index bcff5e3..1a10a53 100644 --- a/test/test_mirah.rb +++ b/test/test_mirah.rb @@ -8,6 +8,7 @@ class TestParsing < Test::Unit::TestCase java_import 'org.mirahparser.mmeta.BaseParser' java_import 'mirahparser.impl.MirahParser' java_import 'mirahparser.lang.ast.NodeScanner' + java_import 'mirahparser.lang.ast.Modifier' java_import 'mirahparser.lang.ast.StringCodeSource' class AstPrinter < NodeScanner @@ -21,9 +22,21 @@ def enterNullChild(obj) def enterDefault(node, arg) @out << ", " unless @first @first = false - @out << "[" << node.java_class.simple_name + if node.kind_of? Modifier + @out << "[" << node.java_class.simple_name << ":" << node.value + else + @out << "[" << node.java_class.simple_name + end true end + + def enterModifier(node, arg) + @out << ", " unless @first + @first = false + @out << "[" << node.java_class.simple_name << ":" << node.value + true + end + def exitDefault(node, arg) @first = false @out << "]" @@ -69,18 +82,45 @@ def exitFieldAccess(node, arg) alias exitFieldAssign exitFieldAccess end - def parse(text) + def parse(text, debug=false) @count ||= 0 filename = "#{self.class.name}-#{@count += 1}" - MirahParser.new.parse(StringCodeSource.new(filename, text)) + mirah_parser = MirahParser.new + MirahParser.tracing = debug + result = mirah_parser.parse(StringCodeSource.new(filename, text)) + puts BaseParser.print_r(result) if debug + result + end + + def parse(text, debug=false, &block) + @count ||= 0 + filename = "#{self.class.name}-#{@count += 1}" + mirah_parser = MirahParser.new + if @parser_options + mirah_parser.instance_eval &@parser_options + end + yield mirah_parser if block_given? + MirahParser.tracing = debug + result = mirah_parser.parse(StringCodeSource.new(filename, text)) + puts BaseParser.print_r(result) if debug + result end - def assert_parse(expected, text) - ast = parse(text) + def assert_parse(expected, text, debug=false) + ast = parse(text, debug) str = AstPrinter.new.scan(ast, ast) assert_equal(expected, str, "expected '#{text}' to be converted") end + def parser_options &block + @parser_options = block + end + + def with_options &block + instance_eval &block + @parser_options = nil + end + def assert_fails(text) begin fail("Should raise syntax error, but got #{parse text}") @@ -333,13 +373,32 @@ def test_primary assert_parse("[Script, [[Not, []]]]", '!()') assert_parse("[Script, [[Not, [[Boolean, true]]]]]", '!(true)') assert_parse("[Script, [[ClassAppendSelf, [[Fixnum, 1]]]]]", 'class << self;1;end') - assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList]]]]", 'class A;1;end') - # assert_parse("[Script, [[ClassDefinition, [Colon2, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]]], [Fixnum, 1], [TypeNameList], [AnnotationList]]]]", 'class A::B;1;end') - assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]], [[Fixnum, 1]], [TypeNameList], [AnnotationList]]]]", 'class A < B;1;end') + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList], [ModifierList], null]]]", 'class A;1;end') + # assert_parse("[Script, [[ClassDefinition, [Colon2, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]]], [Fixnum, 1], [TypeNameList], [AnnotationList], [ModifierList], null]]]", 'class A::B;1;end') + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]], [[Fixnum, 1]], [TypeNameList], [AnnotationList], [ModifierList], null]]]", 'class A < B;1;end') assert_parse("[Script, [[FunctionalCall, [SimpleString, foo], [], [Block, null, [[VCall, [SimpleString, x]]]]]]]", "foo do;x;end") assert_parse("[Script, [[FunctionalCall, [SimpleString, foo], [], [Block, null, [[VCall, [SimpleString, y]]]]]]]", "foo {y}") assert_parse("[Script, [[FunctionalCall, [SimpleString, foo?], [], [Block, null, [[VCall, [SimpleString, z]]]]]]]", "foo? {z}") - assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, a]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList]]]]", 'class a;1;end') + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, a]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList], [ModifierList], null]]]", 'class a;1;end') + end + + def test_modifiers + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [], [TypeNameList], [AnnotationList, [Annotation, [Constant, [SimpleString, T]], [HashEntryList]]], [ModifierList, [Modifier:ABSTRACT]], null]]]", "$T\nabstract class A;end") + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [], [TypeNameList], [AnnotationList], [ModifierList, [Modifier:ABSTRACT]], null]]]", "abstract class A;end") + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [], [TypeNameList], [AnnotationList], [ModifierList, [Modifier:ABSTRACT], [Modifier:PRIVATE]], null]]]", "abstract private class A;end") + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], null, [], [TypeNameList], [AnnotationList], [ModifierList, [Modifier:FINAL], [Modifier:ABSTRACT], [Modifier:PRIVATE]], null]]]", "final abstract private class A;end") + assert_parse("[Script, [[MethodDefinition, [SimpleString, +], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList, [Modifier:ABSTRACT]], null]]]", + "abstract def +; 1; end") + assert_parse("[Script, [[StaticMethodDefinition, [SimpleString, puts], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [], [AnnotationList], [ModifierList, [Modifier:FINAL], [Modifier:PROTECTED]], null]]]", + "final protected def self.puts; end") + assert_parse("[Script, [[StaticMethodDefinition, [SimpleString, puts], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [], [AnnotationList], [ModifierList, [Modifier:FINAL], [Modifier:SYNCHRONIZED]], null]]]", + "final synchronized def self.puts; end") + assert_parse("[Script, [[ConstantAssign, [SimpleString, A], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList, [Modifier:FINAL]]]]]", "final A = b") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList, [Modifier:PROTECTED]]]]]", "protected @a = b") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList, [Modifier:FINAL], [Modifier:TRANSIENT]], static]]]", "final transient @@a = b") + assert_fails("abstract") + assert_fails("def abstract;end") + assert_fails("self.abstract") end def test_if @@ -442,56 +501,56 @@ def test_def names = %w(foo bar? baz! def= rescue Class & | ^ < > + - * / % ! ~ <=> == === =~ !~ <= >= << <<< >> != ** []= [] +@ -@) names.each do |name| - assert_parse("[Script, [[MethodDefinition, [SimpleString, #{name}], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, #{name}], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def #{name}; 1; end") - assert_parse("[Script, [[StaticMethodDefinition, [SimpleString, #{name}], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[StaticMethodDefinition, [SimpleString, #{name}], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def self.#{name}; 1; end") end - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 2]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 2]], [AnnotationList], [ModifierList], null]]]", "def foo(a); 2; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo a; 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], [Constant, [SimpleString, String]]]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], [Constant, [SimpleString, String]]]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a:String); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null], [RequiredArgument, [SimpleString, b], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null], [RequiredArgument, [SimpleString, b], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a, b); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], null, [Fixnum, 1]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], null, [Fixnum, 1]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a = 1); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], [SimpleString, int], [Fixnum, 1]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], [SimpleString, int], [Fixnum, 1]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a:int = 1); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], null, [Fixnum, 1]], [OptionalArgument, [SimpleString, b], null, [Fixnum, 2]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, a], null, [Fixnum, 1]], [OptionalArgument, [SimpleString, b], null, [Fixnum, 2]]], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a = 1, b=2); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, null, null], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, null, null], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(*); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, a], null], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, a], null], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(*a); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, a], [Constant, [SimpleString, Object]]], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, a], [Constant, [SimpleString, Object]]], [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(*a:Object); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], [BlockArgument, [SimpleString, a], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], [BlockArgument, [SimpleString, a], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(&a); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], [BlockArgument, optional, [SimpleString, a], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], [BlockArgument, optional, [SimpleString, a], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(&a = nil); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a, b=1, *c, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a, *c, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], null, [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], null, [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a, b=1, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a, b=1, *c, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(b=1, *c, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], null, [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], null, [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(b=1, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList, [OptionalArgument, [SimpleString, b], null, [Fixnum, 1]]], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(b=1, *c, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList, [RequiredArgument, [SimpleString, d], null]], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(*c, d, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], [RestArgument, [SimpleString, c], null], [RequiredArgumentList], [BlockArgument, [SimpleString, e], null]], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(*c, &e); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], [SimpleString, int], [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], [SimpleString, int], [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a):int; 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, bar], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], [SimpleString, int], [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, bar], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], [SimpleString, int], [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def bar:int; 1; end") assert_fails("def foo(*a, *b);end") assert_fails("def foo(&a, &b);end") @@ -537,9 +596,9 @@ def test_command def test_lhs assert_parse("[Script, [[LocalAssignment, [SimpleString, a], [VCall, [SimpleString, b]]]]]", "a = b") - assert_parse("[Script, [[ConstantAssign, [SimpleString, A], [VCall, [SimpleString, b]], [AnnotationList]]]]", "A = b") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList]]]]", "@a = b") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList], static]]]", "@@a = b") + assert_parse("[Script, [[ConstantAssign, [SimpleString, A], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList]]]]", "A = b") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList]]]]", "@a = b") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [VCall, [SimpleString, b]], [AnnotationList], [ModifierList], static]]]", "@@a = b") assert_parse("[Script, [[ElemAssign, [VCall, [SimpleString, a]], [[Fixnum, 0]], [VCall, [SimpleString, b]]]]]", "a[0] = b") assert_parse("[Script, [[AttrAssign, [VCall, [SimpleString, a]], [SimpleString, foo], [VCall, [SimpleString, b]]]]]", "a.foo = b") assert_parse("[Script, [[AttrAssign, [VCall, [SimpleString, a]], [SimpleString, foo], [VCall, [SimpleString, b]]]]]", "a::foo = b") @@ -554,7 +613,7 @@ def test_arg "a &&= b") assert_parse("[Script, [[If, [LocalAccess, [SimpleString, a]], [[LocalAccess, [SimpleString, a]]], [[LocalAssignment, [SimpleString, a], [VCall, [SimpleString, b]]]]]]]", "a ||= b") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Call, [FieldAccess, [SimpleString, a]], [SimpleString, +], [[Fixnum, 1]], null], [AnnotationList]]]]", + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Call, [FieldAccess, [SimpleString, a]], [SimpleString, +], [[Fixnum, 1]], null], [AnnotationList], [ModifierList]]]]", "@a += 1") assert_parse("[Script, [[[LocalAssignment, [SimpleString, $ptemp$1], [VCall, [SimpleString, a]]]," + " [LocalAssignment, [SimpleString, $ptemp$2], [Fixnum, 1]]," + @@ -672,35 +731,35 @@ def test_command def test_macros assert_parse("[Script, [[Unquote, [VCall, [SimpleString, x]]]]]", '`x`') - assert_parse("[Script, [[ClassDefinition, [Unquote, [Constant, [SimpleString, A]]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList]]]]", 'class `A`;1;end') - assert_parse("[Script, [[MethodDefinition, [Unquote, [VCall, [SimpleString, foo]]], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[ClassDefinition, [Unquote, [Constant, [SimpleString, A]]], null, [[Fixnum, 1]], [TypeNameList], [AnnotationList], [ModifierList], null]]]", 'class `A`;1;end') + assert_parse("[Script, [[MethodDefinition, [Unquote, [VCall, [SimpleString, foo]]], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def `foo`(a); 1; end") - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [Unquote, [VCall, [SimpleString, a]]], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [Unquote, [VCall, [SimpleString, a]]], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(`a`); 1; end") assert_parse("[Script, [[Call, [VCall, [SimpleString, a]], [Unquote, [VCall, [SimpleString, foo]]], [], null]]]", 'a.`foo`') assert_parse("[Script, [[Call, [Self], [Unquote, [VCall, [SimpleString, foo]]], [], null]]]", 'self.`foo`') assert_parse("[Script, [[FieldAccess, [Unquote, [VCall, [SimpleString, a]]]]]]", "@`a`") - assert_parse("[Script, [[FieldAssign, [Unquote, [VCall, [SimpleString, a]]], [Fixnum, 1], [AnnotationList]]]]", "@`a` = 1") + assert_parse("[Script, [[FieldAssign, [Unquote, [VCall, [SimpleString, a]]], [Fixnum, 1], [AnnotationList], [ModifierList]]]]", "@`a` = 1") assert_parse("[Script, [[UnquoteAssign, [Unquote, [VCall, [SimpleString, a]]], [VCall, [SimpleString, b]]]]]", "`a` = b") assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null," + - " [[FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, bar]]]]]], [AnnotationList]]]]", + " [[FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, bar]]]]]], [AnnotationList], null]]]", "macro def foo; quote {bar}; end") assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null," + - " [[FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, bar]]]]]], [AnnotationList]]]]", + " [[FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, bar]]]]]], [AnnotationList], null]]]", "macro def foo; quote do bar end; end") assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null," + " [[VCall, [SimpleString, bar]]," + - " [FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, baz]]]]]], [AnnotationList]]]]", + " [FunctionalCall, [SimpleString, quote], [], [Block, null, [[VCall, [SimpleString, baz]]]]]], [AnnotationList], null]]]", "macro def foo; bar; quote do baz end; end") end def test_annotation - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList]]]]]]", "$Foo @a = 1") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, value], [Constant, [SimpleString, Bar]]]]]]]]]", "$Foo[Bar] @a = 1") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, foo], [Constant, [SimpleString, Bar]]]]]]]]]", "$Foo[foo: Bar] @a = 1") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Colon2, [Constant, [SimpleString, foo]], [Constant, [SimpleString, Bar]]], [HashEntryList]]]]]]", "$foo.Bar @a = 1") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Colon2, [Constant, [SimpleString, foo]], [Constant, [SimpleString, Bar]]], [HashEntryList]]]]]]", "$foo::Bar @a = 1") - assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, value], [Array, [[Constant, [SimpleString, Bar]], [Constant, [SimpleString, Baz]]]]]]]]]]]", "$Foo[Bar, Baz] @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList]]], [ModifierList]]]]", "$Foo @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, value], [Constant, [SimpleString, Bar]]]]]], [ModifierList]]]]", "$Foo[Bar] @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, foo], [Constant, [SimpleString, Bar]]]]]], [ModifierList]]]]", "$Foo[foo: Bar] @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Colon2, [Constant, [SimpleString, foo]], [Constant, [SimpleString, Bar]]], [HashEntryList]]], [ModifierList]]]]", "$foo.Bar @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Colon2, [Constant, [SimpleString, foo]], [Constant, [SimpleString, Bar]]], [HashEntryList]]], [ModifierList]]]]", "$foo::Bar @a = 1") + assert_parse("[Script, [[FieldAssign, [SimpleString, a], [Fixnum, 1], [AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList, [HashEntry, [SimpleString, value], [Array, [[Constant, [SimpleString, Bar]], [Constant, [SimpleString, Baz]]]]]]]], [ModifierList]]]]", "$Foo[Bar, Baz] @a = 1") end def test_return @@ -732,13 +791,13 @@ def test_parent end def test_enddef - assert_parse("[Script, [[If, [Fixnum, 1], [[Fixnum, 2]], []], [MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[If, [Fixnum, 1], [[Fixnum, 2]], []], [MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "if 1 then 2; end def foo; 1; end") end def test_array_type - assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], [TypeRefImpl, String, array]]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MethodDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], [TypeRefImpl, String, array]]], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [[Fixnum, 1]], [AnnotationList], [ModifierList], null]]]", "def foo(a:String[]); 1; end") end @@ -746,18 +805,18 @@ def test_interface assert_parse("[Script, [[InterfaceDeclaration, " + "[Constant, [SimpleString, A]], null, " + "[[Fixnum, 1]], " + - "[TypeNameList], [AnnotationList]]]]", + "[TypeNameList], [AnnotationList], [ModifierList], null]]]", "interface A;1;end") assert_parse("[Script, [[InterfaceDeclaration, " + "[Constant, [SimpleString, A]], null, " + "[[Fixnum, 1]], " + "[TypeNameList, [Constant, [SimpleString, B]], [Constant, [SimpleString, C]]], " + - "[AnnotationList]]]]", + "[AnnotationList], [ModifierList], null]]]", "interface A < B, C do 1;end") assert_parse("[Script, [[InterfaceDeclaration, " + "[Constant, [SimpleString, A]], null, [], " + "[TypeNameList], " + - "[AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList]]]]]]", + "[AnnotationList, [Annotation, [Constant, [SimpleString, Foo]], [HashEntryList]]], [ModifierList], null]]]", "$Foo interface A; end") end @@ -785,13 +844,13 @@ def test_package end def test_macro - assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null, [[Fixnum, 1]], [AnnotationList], null]]]", "defmacro foo; 1; end") # assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null, [[Fixnum, 1]], [AnnotationList]]]]", # "defmacro foo do; 1; end") - assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null, [[Fixnum, 1]], [AnnotationList]]]]", + assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], null, [[Fixnum, 1]], [AnnotationList], null]]]", "macro def foo; 1; end") - assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], [[Fixnum, 2]], [AnnotationList]]]]", + assert_parse("[Script, [[MacroDefinition, [SimpleString, foo], [Arguments, [RequiredArgumentList, [RequiredArgument, [SimpleString, a], null]], [OptionalArgumentList], null, [RequiredArgumentList], null], [[Fixnum, 2]], [AnnotationList], null]]]", "macro def foo(a); 2; end") end @@ -862,7 +921,7 @@ def test_unquote_arguments end def test_implements - assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]], [[Fixnum, 1]], [TypeNameList, [Constant, [SimpleString, Bar]]], [AnnotationList]]]]", + assert_parse("[Script, [[ClassDefinition, [Constant, [SimpleString, A]], [Constant, [SimpleString, B]], [[Fixnum, 1]], [TypeNameList, [Constant, [SimpleString, Bar]]], [AnnotationList], [ModifierList], null]]]", "class A < B\n#foo\nimplements Bar;1;end") end @@ -962,4 +1021,24 @@ def test_assign_not_pipes assert_parse "[Script, [[LocalAssignment, [SimpleString, a], [[LocalAssignment, [SimpleString, $or$1], [Not, [VCall, [SimpleString, a]]]], [If, [LocalAccess, [SimpleString, $or$1]], [[LocalAccess, [SimpleString, $or$1]]], [[VCall, [SimpleString, a]]]]]]]]", 'a = !a || a' end -end + + def test_java_doc + with_options do + parser_options do + skip_java_doc false + end + + assert_parse "[Script, [[MethodDefinition, [SimpleString, a], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [], [AnnotationList], [ModifierList], [JavaDoc]]]]", + '/** jdoc */ def a;end' + assert_parse "[Script, [[MethodDefinition, [SimpleString, a], [Arguments, [RequiredArgumentList], [OptionalArgumentList], null, [RequiredArgumentList], null], null, [], [AnnotationList, [Annotation, [Constant, [SimpleString, Anno]], [HashEntryList]]], [ModifierList], [JavaDoc]]]]", + "/** jdoc */ $Anno\n def a;end" + assert_parse "[Script, [[ClassDefinition, [Constant, [SimpleString, a]], null, [], [TypeNameList], [AnnotationList], [ModifierList], [JavaDoc]]]]", + "/** jdoc */ \nclass a;end" + assert_parse "[Script, [[ClassDefinition, [Constant, [SimpleString, a]], null, [], [TypeNameList], [AnnotationList, [Annotation, [Constant, [SimpleString, Anno]], [HashEntryList]]], [ModifierList], [JavaDoc]]]]", + "/** jdoc */\n $Anno \n class a;end" + assert_parse "[Script, [[MacroDefinition, [SimpleString, a], null, [], [AnnotationList], [JavaDoc]]]]", + '/** jdoc */ macro def a;end' + end + end + +end \ No newline at end of file diff --git a/tools/gen_authors.sh b/tools/gen_authors.sh old mode 100755 new mode 100644