Merge pull request #2 from chedim/feature/priority-sorting

Feature/priority sorting

build.gradle

@@ -15,7 +15,7 @@ plugins {
 }
 
 project.group = 'com.onkiup'
-project.version = '0.6.1'
+project.version = '0.7'
 
 compileJava   {
   sourceCompatibility = '1.8'

src/main/java/com/onkiup/linker/parser/ParserLocation.java

@@ -1,40 +1,79 @@
 package com.onkiup.linker.parser;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class ParserLocation {
-  private final String sourceName;
-  private final String source;
-  private final int line, row;
 
-  public ParserLocation(String sourceName, String source, int line, int row) {
-    this.sourceName = sourceName;
-    this.source = source;
+  private static final Logger logger = LoggerFactory.getLogger(ParserLocation.class);
+  public static ParserLocation ZERO = new ParserLocation("unknown", 0,0,0);
+
+  private final int line, column, position;
+  private final String name;
+
+  public ParserLocation(String name, int position, int line, int column) {
+    if (position < 0) {
+      throw new IllegalArgumentException("Position cannot be negative");
+    }
+
+    if (line < 0) {
+      throw new IllegalArgumentException("Line cannot be negative");
+    }
+
+    if (column < 0) {
+      throw new IllegalArgumentException("Column cannot be negative");
+    }
+
+    this.name = name;
+    this.position = position;
     this.line = line;
-    this.row = row;
+    this.column = column;
   }
 
-  public String sourceName() {
-    return sourceName;
+  public String name() {
+    return name;
   }
 
-  public String source() {
-    return source;
+  public int position() {
+    return position;
   }
 
   public int line() {
     return line;
   }
 
-  public int row() {
-    return row;
+  public int column() {
+    return column;
   }
 
   @Override
   public String toString() {
-    return new StringBuilder().append(sourceName == null ? "" : sourceName + ':')
+    return new StringBuilder()
+      .append(name)
+      .append(" - ")
       .append(line)
       .append(':')
-      .append(row)
+      .append(column)
       .toString();
   }
+
+
+  public ParserLocation advance(CharSequence source) {
+    int position = this.position + source.length();
+    int line = this.line;
+    int column = this.column;
+    for (int i = 0; i < source.length(); i++) {
+      if (source.charAt(i) == '\n') {
+        line++;
+        column = 0;
+      } else {
+        column++;
+      }
+    }
+
+    ParserLocation result = new ParserLocation(name, position, line, column);
+    logger.debug("Advanced from {} to {} using chars: '{}'", this, result, source);
+    return result;
+  }
 }
 

src/main/java/com/onkiup/linker/parser/Rule.java

@@ -3,7 +3,10 @@
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.onkiup.linker.parser.token.CollectionToken;
 import com.onkiup.linker.parser.token.PartialToken;
+import com.onkiup.linker.parser.token.RuleToken;
+import com.onkiup.linker.parser.token.VariantToken;
 
 // in 0.4:
 // - changed Metadata to hold PartialTokens instead of ParserLocations
@@ -22,7 +25,7 @@ public static Optional<PartialToken> metadata(Rule rule) {
       return Optional.ofNullable(metadata.get(rule));
     }
 
-    static void metadata(Rule rule, PartialToken token) {
+    public static void metadata(Rule rule, PartialToken token) {
       metadata.put(rule, token);
     }
 
@@ -35,9 +38,15 @@ static void remove(Rule rule) {
    * @returns parent token or null if this token is root token
    */
   default <R extends Rule> Optional<R> parent() {
-    return Metadata.metadata(this)
-      .flatMap(PartialToken::getParent)
-      .map(parent -> (R) ((PartialToken)parent).getToken());
+    PartialToken meta = Metadata.metadata(this).get();
+    do {
+      meta = (PartialToken) meta.getParent().orElse(null);
+    } while (meta != null && !(meta instanceof RuleToken));
+
+    if (meta != null) {
+      return Optional.of((R) meta.getToken());
+    }
+    return Optional.empty();
   }
 
   /**
@@ -49,12 +58,24 @@ default boolean populated() {
       .orElse(false);
   }
 
+  default PartialToken metadata() {
+    return Metadata.metadata(this).get();
+  }
+
   /**
    * Reevaluation callback.
    * Called by parser every time it updates the token
    */
   default void reevaluate() {
 
   }
+
+  /**
+   * Invalidation callback
+   * called by arser every time it detaches the token from the tree
+   */
+  default void invalidate() {
+
+  }
 }
 

src/main/java/com/onkiup/linker/parser/SyntaxError.java

@@ -1,8 +1,46 @@
 package com.onkiup.linker.parser;
 
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.onkiup.linker.parser.token.PartialToken;
+import com.onkiup.linker.parser.token.RuleToken;
+import com.onkiup.linker.parser.token.VariantToken;
+
 public class SyntaxError extends RuntimeException {
-  public SyntaxError(String message) {
-    super(message);
+
+  private PartialToken lastToken;
+  private StringBuilder source;
+  private String message;
+
+  public SyntaxError(String message, PartialToken lastToken, StringBuilder source) {
+    this.message = message;
+    this.lastToken = lastToken;
+    this.source = source;
+  }
+
+  @Override
+  public String toString() {
+    PartialToken expected = lastToken.expected();
+    StringBuilder result = new StringBuilder("Parser error:")
+      .append(message)
+      .append("\n")
+      .append("\tExpected ")
+      .append(expected)
+      .append(" but got: '")
+      .append(expected != null && source != null && expected.position() < source.length() ? source.substring(expected.position()) : source)
+      .append("'\n\tSource:\n\t\t")
+      .append(source)
+      .append("\n\n\tTraceback:\n\t\t");
+
+    PartialToken parent = expected;
+    while (null != (parent = (PartialToken) parent.getParent().orElse(null))) {
+      result.append(parent.toString().replace("\n", "\n\t\t"));
+    }
+
+    return result.toString();
   }
 }
 

src/main/java/com/onkiup/linker/parser/TerminalMatcher.java

@@ -1,6 +1,7 @@
 package com.onkiup.linker.parser;
 
 public class TerminalMatcher implements TokenMatcher {
+
   private final String pattern; 
   private final int patternLen;
 

src/main/java/com/onkiup/linker/parser/TokenGrammar.java

@@ -4,6 +4,7 @@
 import java.io.StringReader;
 import java.lang.reflect.Modifier;
 import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
@@ -36,15 +37,18 @@ public Class<X> getTokenType() {
   }
 
   public X parse(String source) throws SyntaxError {
-    return parse(new StringReader(source));
+    return parse("unknown", source);
+  }
+  public X parse(String name, String source) throws SyntaxError {
+    return parse(name, new StringReader(source));
   }
 
   public X parse(Reader source) throws SyntaxError {
-    return parse(source, null);
+    return parse("unknown", source);
   }
 
-  public X parse(Reader source, Object context)  throws SyntaxError {
-    X result = tokenize(source);
+  public X parse(String name, Reader source)  throws SyntaxError {
+    X result = tokenize(name, source);
     StringBuilder tail = new StringBuilder();
     try {
       int nextChar;
@@ -56,23 +60,28 @@ public X parse(Reader source, Object context)  throws SyntaxError {
     }
 
     if (tail.length() > 0) {
-      throw new SyntaxError("Unmatched trailing symbols: '" + tail + "'");
+      throw new SyntaxError("Unmatched trailing symbols: '" + tail + "'", null, tail);
     }
     return result;
   }
 
   public X tokenize(Reader source) throws SyntaxError {
-    return tokenize("", source);
+    return tokenize("unknown", source);
   }
 
   public X tokenize(String sourceName, Reader source) throws SyntaxError {
-    PartialToken<X> rootToken = PartialToken.forClass(null, type, 0);
+    PartialToken<X> rootToken = PartialToken.forClass(null, type, new ParserLocation(sourceName, 0, 0, 0));
     PartialToken token = rootToken;
+    PartialToken lastToken = token;
     StringBuilder buffer = new StringBuilder();
     boolean hitEnd = false;
-    int position = 0, line = 0, col = 0;
+    int line = 0, col = 0;
+    AtomicInteger position = new AtomicInteger(0); 
     try {
       do {
+        logger.debug("----------------------------------------------------------------------------------------");
+        logger.debug("BUFFER = '{}'", buffer);
+        logger.debug("POSITION = {}", position.get());
         logger.debug("----------------------------------------------------------------------------------------");
         if (logger.isDebugEnabled()) {
           Collection<PartialToken> path = token.getPath();
@@ -89,7 +98,7 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError {
           if (!hitEnd) {
             buffer.append((char) nextChar);
 
-            position++;
+            position.getAndIncrement();
             col++;
 
             if (nextChar == '\n') {
@@ -105,13 +114,15 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError {
           boolean consumed = (Boolean)consumingToken.consume(buffer.charAt(0), buffer.length() == 1)
             .map(Object::toString)
             .map(returned -> {
-              logger.debug("Token {} returned characters: {}", consumingToken, returned);
               buffer.replace(0, 1, (String) returned);
+              position.addAndGet(1 - ((String)returned).length());
+              logger.debug("Token {} returned characters: '{}'; buffer = '{}'", consumingToken, returned, buffer);
               return false;
             })
             .orElseGet(() -> {
               logger.debug("Discarding consumed by token {} character '{}'", consumingToken, buffer.charAt(0));
               buffer.delete(0, 1);
+              position.incrementAndGet();
               return true;
             });
 
@@ -121,6 +132,8 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError {
           }
         }
 
+        lastToken = token;
+
         do {
           token = (PartialToken) token.advance(buffer.length() == 0).orElse(null);
           logger.debug("Advanced to token {}", token);
@@ -135,13 +148,16 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError {
         if (token == null) {
           if (buffer.length() > 0 || !hitEnd) {
             logger.debug("Trying to rotate root token to avoid unmatched characters...");
-            if (rootToken.rotate()) {
+            if (rootToken.rotatable()) {
+              rootToken.rotate();
               logger.debug("Rotated root token");
               token = rootToken;
               continue;
             }
 
-            if (rootToken.alternativesLeft() > 0) {
+            int alternativesLeft = rootToken.alternativesLeft();
+            logger.debug("Alternatives left for {}: {}", rootToken, alternativesLeft);
+            if (alternativesLeft > 0) {
               logger.debug("Hit end but root token had alternatives left; backtracking the token and continuing with next variant");
               rootToken.pullback().ifPresent(b -> buffer.insert(0, b));
               token = rootToken;
@@ -152,19 +168,21 @@ public X tokenize(String sourceName, Reader source) throws SyntaxError {
             while (-1 != (nextChar = source.read())) {
               buffer.append((char)nextChar);
             }
-            throw new SyntaxError("Unmatched trailing characters: " + buffer);
+            throw new SyntaxError("Unmatched trailing characters", rootToken, buffer);
           } else {
+            rootToken.sortPriorities();
             return rootToken.getToken();
           }
         }
       } while(buffer.length() > 0);
 
-      throw new SyntaxError("Unexpected end of input");
+      throw new SyntaxError("Unexpected end of input", rootToken, buffer);
     } catch (SyntaxError se) {
       throw new RuntimeException("Syntax error at line " + line + ", column " + col, se);
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
+
 }
 

src/main/java/com/onkiup/linker/parser/annotation/AdjustPriority.java

@@ -9,5 +9,6 @@
 @Retention(RetentionPolicy.RUNTIME)
 public @interface AdjustPriority {
   int value();
+  boolean propagate() default false;
 }
 

src/main/java/com/onkiup/linker/parser/annotation/Optional.java → src/main/java/com/onkiup/linker/parser/annotation/OptionalToken.java

@@ -7,7 +7,7 @@
 
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.FIELD)
-public @interface Optional {
+public @interface OptionalToken {
   String whenFollowedBy() default "";
 }