Added Lexer Tests

invalid.txt

@@ -0,0 +1 @@
+"Hello World

src/main/java/com/auberer/compilerdesignlectureproject/CompilerDesignLectureProject.java

@@ -1,7 +1,20 @@
 package com.auberer.compilerdesignlectureproject;
 
+import com.auberer.compilerdesignlectureproject.lexer.Lexer;
+
 public class CompilerDesignLectureProject {
+    static String basePath = System.getProperty("user.dir");
+    static String testfile = new String(basePath + "/test.txt");
+    static String emptyfile = new String(basePath + "/empty.txt");
+    static String invalidFile = new String(basePath + "/invalid.txt");
+
     public static void main(String[] args) {
-        System.out.println("Hello World!");
+        try {
+            Lexer lexer = new Lexer(testfile);
+            lexer.start();
+        }
+        catch (Exception ex){
+            ex.printStackTrace();
+        }
     }
 }

src/main/java/com/auberer/compilerdesignlectureproject/lexer/Lexer.java

@@ -9,6 +9,7 @@
 import java.util.Set;
 
 import com.auberer.compilerdesignlectureproject.reader.Reader;
+import lombok.Getter;
 
 /**
  * Input: Character Strean
@@ -18,26 +19,34 @@ public class Lexer implements ILexer {
     private final Reader reader;
     private Token currentToken;
 
+    @Getter
+    private List<Token> tokens = new ArrayList<>();
+
     private List<StateMachine> stateMachineList = new ArrayList<>();
 
     public Lexer(String inputFilePath) throws IOException {
         this.reader = new Reader(inputFilePath);
 
-        stateMachineList.add(new IntegerLiteralStateMachine());
-        stateMachineList.add(new KeywordStateMachine("DankeRoethig"));
+        stateMachineList.add(new IdentifierStateMachine());
         stateMachineList.add(new StringLiteralStateMachine());
-        stateMachineList.add(new KeywordStateMachine("XSLT"));
-        stateMachineList.add(new KeywordStateMachine("XML"));
-        stateMachineList.add(new DoubleLiteralStateMachine());
         stateMachineList.add(new IntegerLiteralStateMachine());
+        stateMachineList.add(new DoubleLiteralStateMachine());
+        stateMachineList.add(new KeywordStateMachine("int"));
+        stateMachineList.add(new KeywordStateMachine("double"));
+        stateMachineList.add(new KeywordStateMachine("String"));
 
         for (StateMachine stateMachine : stateMachineList) {
             stateMachine.init();
             stateMachine.reset();
         }
 
-        // Read the 1st token
-        advance();
+    }
+
+    public void start() throws IOException {
+        do {
+            advance();
+        }
+        while(!isEOF());
     }
 
     @Override
@@ -51,7 +60,10 @@ public void advance() throws IOException {
         while (Character.isWhitespace(reader.getChar())) {
             reader.advance();
         }
-        currentToken = consumeToken();
+        Token token = consumeToken();
+
+        currentToken = token;
+        tokens.add(token);
     }
 
     @Override
@@ -87,47 +99,49 @@ private Token consumeToken() throws IOException {
         }
 
         for (StateMachine stateMachine : stateMachineList) {
-            stateMachine.init();
             stateMachine.reset();
         }
 
         List<StateMachine> runningMachines = new ArrayList<>(stateMachineList);
         StateMachine acceptingMachine = null;
         StringBuilder currentToken = new StringBuilder();
 
+        CodeLoc loc = reader.getCodeLoc();
+        loc = new CodeLoc(loc.getLine(), loc.getColumn());
+
         while(!runningMachines.isEmpty()){
+            if(reader.isEOF()){
+                break;
+            }
+
+            char currentChar = reader.getChar();
+            if(currentChar == '\r' || currentChar == '\n'){
+                break;
+            }
+
+            currentToken.append(currentChar);
+            List<StateMachine> endedMachines = new ArrayList<>();
             for(StateMachine stateMachine: runningMachines){
-                currentToken.append(reader.getChar());
                 try {
-                    stateMachine.processInput(reader.getChar());
+                    stateMachine.processInput(currentChar);
                 }
                 catch(IllegalStateException ex){
-                    runningMachines.remove(stateMachine);
+                    endedMachines.add(stateMachine);
                 }
                 if(stateMachine.isInAcceptState()){
                     acceptingMachine = stateMachine;
-                    runningMachines.remove(stateMachine);
                 }
             }
+            runningMachines.removeAll(endedMachines);
+            endedMachines.clear();
             reader.advance();
         }
 
-        if(acceptingMachine instanceof IntegerLiteralStateMachine){
-            return new Token(TokenType.TOK_INTEGER, currentToken.toString(), reader.getCodeLoc());
-        }
-        else if(acceptingMachine instanceof DoubleLiteralStateMachine){
-            return new Token(TokenType.TOK_DOUBLE, currentToken.toString(), reader.getCodeLoc());
-        }
-        else if(acceptingMachine instanceof StringLiteralStateMachine){
-            return new Token(TokenType.TOK_STRING, currentToken.toString(), reader.getCodeLoc());
-        }
-        else if(acceptingMachine instanceof IdentifierStateMachine){
-            return new Token(TokenType.TOK_IDENTIFIER, currentToken.toString(), reader.getCodeLoc());
-        }
-        else if(acceptingMachine instanceof KeywordStateMachine){
-            return new Token(TokenType.TOK_KEYWORD, currentToken.toString(), reader.getCodeLoc());
+        if(acceptingMachine == null){
+            return new Token(TokenType.TOK_INVALID, currentToken.toString(), loc);
         }
 
-        return null;
+        return new Token(acceptingMachine.getTokenType(), currentToken.toString(), loc);
     }
+
 }

src/main/java/com/auberer/compilerdesignlectureproject/reader/Reader.java

@@ -40,7 +40,7 @@ public void advance() throws IOException {
         } else {
             if (currentChar == '\n') {
                 codeLoc.line++;
-                codeLoc.column = 0;
+                codeLoc.column = 1;
             } else {
                 codeLoc.column++;
             }

src/test/java/com/auberer/compilerdesignlectureproject/lexer/LexerTest.java

@@ -1,4 +1,51 @@
 package com.auberer.compilerdesignlectureproject.lexer;
 
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 public class LexerTest {
+
+    static String basePath = System.getProperty("user.dir");
+    static String testfile = new String(basePath + "/test.txt");
+    static String emptyfile = new String(basePath + "/empty.txt");
+    static String invalidFile = new String(basePath + "/invalid.txt");
+
+    @Test
+    @DisplayName("Test for End of file")
+    public void isEOF() throws IOException {
+        Lexer lexer = new Lexer(emptyfile);
+        lexer.start();
+
+        assertEquals(lexer.getTokens().size(), 1);
+        assertEquals(lexer.getTokens().get(0).getType(), TokenType.TOK_EOF);
+    }
+
+    @Test
+    @DisplayName("Test for invalid token")
+    public void isInvalid() throws IOException {
+        Lexer lexer = new Lexer(invalidFile);
+        lexer.start();
+
+        assertEquals(lexer.getTokens().get(0).getType(), TokenType.TOK_INVALID);
+    }
+
+    @Test
+    @DisplayName("Test for valid file")
+    public void isValid() throws IOException{
+        Lexer lexer = new Lexer(testfile);
+        lexer.start();
+
+        assertEquals(lexer.getTokens().size(), 8);
+
+        TokenType[] expected = {TokenType.TOK_KEYWORD, TokenType.TOK_IDENTIFIER, TokenType.TOK_INTEGER, TokenType.TOK_STRING, TokenType.TOK_KEYWORD, TokenType.TOK_IDENTIFIER, TokenType.TOK_DOUBLE, TokenType.TOK_EOF};
+
+        for(int i = 0; i < lexer.getTokens().size(); i++){
+            assertEquals(lexer.getTokens().get(i).getType(), expected[i]);
+        }
+
+    }
 }

src/test/java/com/auberer/compilerdesignlectureproject/reader/ReaderTest.java

@@ -11,12 +11,13 @@
 
 public class ReaderTest {
 
-    static String testfile = new String("compiler-design-lecture-project-TINF22B6/test.txt");
-    static String emptyfile = new String("compiler-design-lecture-project-TINF22B6/test.txt");
+    static String basePath = System.getProperty("user.dir");
+    static String testfile = new String(basePath + "/test.txt");
+    static String emptyfile = new String(basePath + "/empty.txt");
 
 
     @Test
-    @DisplayName("This is a dafault Test")
+    @DisplayName("This is a default Test")
     public void test() {
         System.out.printf("This is a test:%n");
         Assertions.assertEquals("Test", "Test");

test.txt

@@ -1 +1,3 @@
-Irgendwas
+int a 10
+"hello world" double ddd
+22,5