diff --git a/invalid.txt b/invalid.txt new file mode 100644 index 0000000..55e3a44 --- /dev/null +++ b/invalid.txt @@ -0,0 +1 @@ +"Hello World \ No newline at end of file diff --git a/src/main/java/com/auberer/compilerdesignlectureproject/CompilerDesignLectureProject.java b/src/main/java/com/auberer/compilerdesignlectureproject/CompilerDesignLectureProject.java index d30e20c..691e8f4 100644 --- a/src/main/java/com/auberer/compilerdesignlectureproject/CompilerDesignLectureProject.java +++ b/src/main/java/com/auberer/compilerdesignlectureproject/CompilerDesignLectureProject.java @@ -1,7 +1,20 @@ package com.auberer.compilerdesignlectureproject; +import com.auberer.compilerdesignlectureproject.lexer.Lexer; + public class CompilerDesignLectureProject { + static String basePath = System.getProperty("user.dir"); + static String testfile = new String(basePath + "/test.txt"); + static String emptyfile = new String(basePath + "/empty.txt"); + static String invalidFile = new String(basePath + "/invalid.txt"); + public static void main(String[] args) { - System.out.println("Hello World!"); + try { + Lexer lexer = new Lexer(testfile); + lexer.start(); + } + catch (Exception ex){ + ex.printStackTrace(); + } } } diff --git a/src/main/java/com/auberer/compilerdesignlectureproject/lexer/Lexer.java b/src/main/java/com/auberer/compilerdesignlectureproject/lexer/Lexer.java index 858c83c..06e0628 100644 --- a/src/main/java/com/auberer/compilerdesignlectureproject/lexer/Lexer.java +++ b/src/main/java/com/auberer/compilerdesignlectureproject/lexer/Lexer.java @@ -9,6 +9,7 @@ import java.util.Set; import com.auberer.compilerdesignlectureproject.reader.Reader; +import lombok.Getter; /** * Input: Character Strean @@ -18,26 +19,34 @@ public class Lexer implements ILexer { private final Reader reader; private Token currentToken; + @Getter + private List tokens = new ArrayList<>(); + private List stateMachineList = new ArrayList<>(); public Lexer(String inputFilePath) throws IOException { this.reader = new Reader(inputFilePath); - stateMachineList.add(new IntegerLiteralStateMachine()); - stateMachineList.add(new KeywordStateMachine("DankeRoethig")); + stateMachineList.add(new IdentifierStateMachine()); stateMachineList.add(new StringLiteralStateMachine()); - stateMachineList.add(new KeywordStateMachine("XSLT")); - stateMachineList.add(new KeywordStateMachine("XML")); - stateMachineList.add(new DoubleLiteralStateMachine()); stateMachineList.add(new IntegerLiteralStateMachine()); + stateMachineList.add(new DoubleLiteralStateMachine()); + stateMachineList.add(new KeywordStateMachine("int")); + stateMachineList.add(new KeywordStateMachine("double")); + stateMachineList.add(new KeywordStateMachine("String")); for (StateMachine stateMachine : stateMachineList) { stateMachine.init(); stateMachine.reset(); } - // Read the 1st token - advance(); + } + + public void start() throws IOException { + do { + advance(); + } + while(!isEOF()); } @Override @@ -51,7 +60,10 @@ public void advance() throws IOException { while (Character.isWhitespace(reader.getChar())) { reader.advance(); } - currentToken = consumeToken(); + Token token = consumeToken(); + + currentToken = token; + tokens.add(token); } @Override @@ -87,7 +99,6 @@ private Token consumeToken() throws IOException { } for (StateMachine stateMachine : stateMachineList) { - stateMachine.init(); stateMachine.reset(); } @@ -95,39 +106,42 @@ private Token consumeToken() throws IOException { StateMachine acceptingMachine = null; StringBuilder currentToken = new StringBuilder(); + CodeLoc loc = reader.getCodeLoc(); + loc = new CodeLoc(loc.getLine(), loc.getColumn()); + while(!runningMachines.isEmpty()){ + if(reader.isEOF()){ + break; + } + + char currentChar = reader.getChar(); + if(currentChar == '\r' || currentChar == '\n'){ + break; + } + + currentToken.append(currentChar); + List endedMachines = new ArrayList<>(); for(StateMachine stateMachine: runningMachines){ - currentToken.append(reader.getChar()); try { - stateMachine.processInput(reader.getChar()); + stateMachine.processInput(currentChar); } catch(IllegalStateException ex){ - runningMachines.remove(stateMachine); + endedMachines.add(stateMachine); } if(stateMachine.isInAcceptState()){ acceptingMachine = stateMachine; - runningMachines.remove(stateMachine); } } + runningMachines.removeAll(endedMachines); + endedMachines.clear(); reader.advance(); } - if(acceptingMachine instanceof IntegerLiteralStateMachine){ - return new Token(TokenType.TOK_INTEGER, currentToken.toString(), reader.getCodeLoc()); - } - else if(acceptingMachine instanceof DoubleLiteralStateMachine){ - return new Token(TokenType.TOK_DOUBLE, currentToken.toString(), reader.getCodeLoc()); - } - else if(acceptingMachine instanceof StringLiteralStateMachine){ - return new Token(TokenType.TOK_STRING, currentToken.toString(), reader.getCodeLoc()); - } - else if(acceptingMachine instanceof IdentifierStateMachine){ - return new Token(TokenType.TOK_IDENTIFIER, currentToken.toString(), reader.getCodeLoc()); - } - else if(acceptingMachine instanceof KeywordStateMachine){ - return new Token(TokenType.TOK_KEYWORD, currentToken.toString(), reader.getCodeLoc()); + if(acceptingMachine == null){ + return new Token(TokenType.TOK_INVALID, currentToken.toString(), loc); } - return null; + return new Token(acceptingMachine.getTokenType(), currentToken.toString(), loc); } + } diff --git a/src/main/java/com/auberer/compilerdesignlectureproject/reader/Reader.java b/src/main/java/com/auberer/compilerdesignlectureproject/reader/Reader.java index 18d1149..cf4fd2e 100644 --- a/src/main/java/com/auberer/compilerdesignlectureproject/reader/Reader.java +++ b/src/main/java/com/auberer/compilerdesignlectureproject/reader/Reader.java @@ -40,7 +40,7 @@ public void advance() throws IOException { } else { if (currentChar == '\n') { codeLoc.line++; - codeLoc.column = 0; + codeLoc.column = 1; } else { codeLoc.column++; } diff --git a/src/test/java/com/auberer/compilerdesignlectureproject/lexer/LexerTest.java b/src/test/java/com/auberer/compilerdesignlectureproject/lexer/LexerTest.java index 0ec2262..09dc3ec 100644 --- a/src/test/java/com/auberer/compilerdesignlectureproject/lexer/LexerTest.java +++ b/src/test/java/com/auberer/compilerdesignlectureproject/lexer/LexerTest.java @@ -1,4 +1,51 @@ package com.auberer.compilerdesignlectureproject.lexer; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + public class LexerTest { + + static String basePath = System.getProperty("user.dir"); + static String testfile = new String(basePath + "/test.txt"); + static String emptyfile = new String(basePath + "/empty.txt"); + static String invalidFile = new String(basePath + "/invalid.txt"); + + @Test + @DisplayName("Test for End of file") + public void isEOF() throws IOException { + Lexer lexer = new Lexer(emptyfile); + lexer.start(); + + assertEquals(lexer.getTokens().size(), 1); + assertEquals(lexer.getTokens().get(0).getType(), TokenType.TOK_EOF); + } + + @Test + @DisplayName("Test for invalid token") + public void isInvalid() throws IOException { + Lexer lexer = new Lexer(invalidFile); + lexer.start(); + + assertEquals(lexer.getTokens().get(0).getType(), TokenType.TOK_INVALID); + } + + @Test + @DisplayName("Test for valid file") + public void isValid() throws IOException{ + Lexer lexer = new Lexer(testfile); + lexer.start(); + + assertEquals(lexer.getTokens().size(), 8); + + TokenType[] expected = {TokenType.TOK_KEYWORD, TokenType.TOK_IDENTIFIER, TokenType.TOK_INTEGER, TokenType.TOK_STRING, TokenType.TOK_KEYWORD, TokenType.TOK_IDENTIFIER, TokenType.TOK_DOUBLE, TokenType.TOK_EOF}; + + for(int i = 0; i < lexer.getTokens().size(); i++){ + assertEquals(lexer.getTokens().get(i).getType(), expected[i]); + } + + } } diff --git a/src/test/java/com/auberer/compilerdesignlectureproject/reader/ReaderTest.java b/src/test/java/com/auberer/compilerdesignlectureproject/reader/ReaderTest.java index 6592b80..99f4742 100644 --- a/src/test/java/com/auberer/compilerdesignlectureproject/reader/ReaderTest.java +++ b/src/test/java/com/auberer/compilerdesignlectureproject/reader/ReaderTest.java @@ -11,12 +11,13 @@ public class ReaderTest { - static String testfile = new String("compiler-design-lecture-project-TINF22B6/test.txt"); - static String emptyfile = new String("compiler-design-lecture-project-TINF22B6/test.txt"); + static String basePath = System.getProperty("user.dir"); + static String testfile = new String(basePath + "/test.txt"); + static String emptyfile = new String(basePath + "/empty.txt"); @Test - @DisplayName("This is a dafault Test") + @DisplayName("This is a default Test") public void test() { System.out.printf("This is a test:%n"); Assertions.assertEquals("Test", "Test"); diff --git a/src/test/java/com/auberer/compilerdesignlectureproject/reader/empty.txt b/src/test/java/com/auberer/compilerdesignlectureproject/reader/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/test.txt b/test.txt index bd2f4a9..80d2630 100644 --- a/test.txt +++ b/test.txt @@ -1 +1,3 @@ -Irgendwas \ No newline at end of file +int a 10 +"hello world" double ddd +22,5 \ No newline at end of file