diff --git a/lexer/v2/IToken.go b/lexer/IToken.go similarity index 99% rename from lexer/v2/IToken.go rename to lexer/IToken.go index 05972d4..57be485 100644 --- a/lexer/v2/IToken.go +++ b/lexer/IToken.go @@ -1,4 +1,4 @@ -package v2 +package lexer type ITokenType interface { // Resolve use the actual l *TLexer to create a new Token. the lexer is ready for the next step when the resolve end diff --git a/lexer/v2/Lexer.go b/lexer/Lexer.go similarity index 96% rename from lexer/v2/Lexer.go rename to lexer/Lexer.go index 6841e2b..adc9853 100644 --- a/lexer/v2/Lexer.go +++ b/lexer/Lexer.go @@ -1,4 +1,4 @@ -package v2 +package lexer import "strconv" @@ -207,12 +207,9 @@ func LexerR(sentence string) []Token { Lex.Step() Lex.tempVal = Lex.sentence[Lex.prevIndex:Lex.index] if Lex.tempVal != "" { - if Lex.TriggerBy != "" { - //findNameInEveryTokenType(Lex.TriggerBy, Every).Resolve(Lex) - Lex.AddToken(Lex.TriggerBy + " FIN") - } else { - Lex.AddToken(TEXT) - } + + Lex.AddToken(TEXT) + Lex.tempVal = "" } Lex.AddToken(EOF) diff --git a/lexer/v2/TokenTypeBase.go b/lexer/TokenTypeBase.go similarity index 99% rename from lexer/v2/TokenTypeBase.go rename to lexer/TokenTypeBase.go index e391750..0c861a2 100644 --- a/lexer/v2/TokenTypeBase.go +++ b/lexer/TokenTypeBase.go @@ -1,4 +1,4 @@ -package v2 +package lexer type TokenTypeBaseBehavior struct { Name string diff --git a/lexer/v2/TokenTypeComposite.go b/lexer/TokenTypeComposite.go similarity index 99% rename from lexer/v2/TokenTypeComposite.go rename to lexer/TokenTypeComposite.go index 6838ec4..ab1facb 100644 --- a/lexer/v2/TokenTypeComposite.go +++ b/lexer/TokenTypeComposite.go @@ -1,4 +1,4 @@ -package v2 +package lexer type TokenTypeCompositeBehavior struct { Name string diff --git a/lexer/v2/TokenTypeMerger.go b/lexer/TokenTypeMerger.go similarity index 95% rename from lexer/v2/TokenTypeMerger.go rename to lexer/TokenTypeMerger.go index 5f670c3..400fe1b 100644 --- a/lexer/v2/TokenTypeMerger.go +++ b/lexer/TokenTypeMerger.go @@ -1,4 +1,4 @@ -package v2 +package lexer type TokenTypeMergerBehavior struct { Name string @@ -36,7 +36,10 @@ func (t *TokenTypeMergerBehavior) Resolve(l *TLexer) { (*l).isSpaces = false } if index == -1 { - if l.sizeOfTokenReversed != -1 { + if l.index >= len(l.sentence) { + l.AddToken(t.Result[0].Name) + l.prevIndex = l.index + } else if l.sizeOfTokenReversed != -1 { identified := l.tempVal[len(l.tempVal)-l.sizeOfTokenReversed:] indexOfClose := t.IsClosedBySyntaxe(identified) if indexOfClose != -1 { diff --git a/lexer/v2/TokenTypeSpaces.go b/lexer/TokenTypeSpaces.go similarity index 96% rename from lexer/v2/TokenTypeSpaces.go rename to lexer/TokenTypeSpaces.go index 3af0a33..794e3fe 100644 --- a/lexer/v2/TokenTypeSpaces.go +++ b/lexer/TokenTypeSpaces.go @@ -1,4 +1,4 @@ -package v2 +package lexer type TokenTypeSpacesBehavior struct { Name string @@ -19,7 +19,7 @@ func (t *TokenTypeSpacesBehavior) Resolve(l *TLexer) { } if t.Name == "\n" { (*l).line++ - (*l).position = 0 + (*l).position = 1 } l.prevIndex = l.index diff --git a/lexer/v2/TokenTypeTrigger.go b/lexer/TokenTypeTrigger.go similarity index 89% rename from lexer/v2/TokenTypeTrigger.go rename to lexer/TokenTypeTrigger.go index 256e2b4..94fbe7e 100644 --- a/lexer/v2/TokenTypeTrigger.go +++ b/lexer/TokenTypeTrigger.go @@ -1,4 +1,4 @@ -package v2 +package lexer type TokenTypeTriggerBehavior struct { Name string @@ -18,7 +18,10 @@ func (t *TokenTypeTriggerBehavior) Resolve(l *TLexer) { (*l).AddToken(t.Name) l.prevIndex = l.index } else { - if l.sizeOfTokenReversed != -1 { + if l.index >= len(l.sentence) { + l.AddToken(t.Result[0].Name) + l.prevIndex = l.index + } else if l.sizeOfTokenReversed != -1 { identified := l.tempVal[len(l.tempVal)-l.sizeOfTokenReversed:] indexOfClose := t.IsClosedBySyntaxe(identified) if indexOfClose != -1 { @@ -78,10 +81,10 @@ var ( "\"", }, CloseBy: []ITokenType{ - &SELF, + &SELF, &RETURN, }, Result: []TokenTypeCompositeBehavior{ - CSTRING, + CSTRING, CSTRING, }, } TSQUOTE = TokenTypeTriggerBehavior{ @@ -90,10 +93,10 @@ var ( "'", }, CloseBy: []ITokenType{ - &SELF, + &SELF, &RETURN, }, Result: []TokenTypeCompositeBehavior{ - CCHAR, + CCHAR, CCHAR, }, } ) diff --git a/lexer/lexer.go b/lexer/lexer.go deleted file mode 100644 index 83b595d..0000000 --- a/lexer/lexer.go +++ /dev/null @@ -1,486 +0,0 @@ -package lexer - -// Token is a struct that contains all the information about a token -type Token struct { - TokenType string - Value string - Position int - Line int -} - -// Lexer do a lexical analysis of the string sentence to separate each element, -// and associate each element with a token -func Lexer(sentence string) []Token { - - // ret is the []Token that the lexer will return - var ret []Token - - // prevIndex is index of the start of the element that we want to compare - // with the known syntaxe - var prevIndex int = 0 - var actualIndex int = 1 - // line will be increase each time a ";" is founded - var line int = 0 - // canBeText is false when an element is already considered as a known - // syntaxe, and true elsewhere - var canBeText bool - var isSpaces bool - var inQuote bool - var inQuoteStep bool - var QuoteIdentifier string - var endOfComm int = -1 - var endOfCommGroup int = -1 - - // tempVal is the current element that we want to compare with the known - // syntaxe - var tempVal string - - for i := 0; i <= len(sentence); i++ { - // we assign tempVal as an element in the interval [prevIndex:i] - tempVal = sentence[prevIndex:i] - // we assign canBeText to true, because we actually don't know if the - // current element is a text or not - canBeText = true - inQuoteStep = false - - for _, ident := range Identifier { - // -----------Is Known Syntaxe Part------------- - // - // for each element of Identifier, we compare all the known - // syntaxes with our tempVal, If the comparison is true, - // tempVal is a known syntaxes, and then a token - if ident.IsSyntaxe(tempVal) { - - // canot be a text now - canBeText = false - if (ident.Identifier == COMMENT || ident.Identifier == COMMENTGROUP) && inQuote && len(ret) != 0 { - break - } - // -----------Previous Token COMMENT Part------------- - if len(ret) > 1 { - // if the previous token is a COMMENT, we must concat the actual value to the previous - // token instead of create a new one. - // we don't concat only if it's the end of the COMMENT token. - // if we stop concat the value with the COMMENT, we keep the COMMENT token index, - // like that we can be sure to not append something else in. - if ret[len(ret)-1].TokenType == COMMENT && len(ret)-1 != endOfComm && !inQuote { - if tempVal == "/" && ret[len(ret)-1].Value == "" { - ret[len(ret)-1].TokenType = COMMENTGROUP - break - } else if tempVal != "\n" && tempVal != "\r" { - ret[len(ret)-1].Value += tempVal - } else { - endOfComm = len(ret) - 1 - } - tempVal = "" - prevIndex = i - break - } - // same things for the COMMENTGROUP, but with a different ending close. - if ret[len(ret)-1].TokenType == COMMENTGROUP && len(ret)-1 != endOfCommGroup && !inQuote { - if tempVal[len(tempVal)-1] == '/' { - break - } else if len(tempVal) > 1 { - if tempVal[len(tempVal)-2:] == "/#" { - endOfCommGroup = len(ret) - 1 - } else { - ret[len(ret)-1].Value += tempVal - } - } else { - ret[len(ret)-1].Value += tempVal - } - tempVal = "" - prevIndex = i - break - } - } - // -----------Previous Token COMMENT Part END------------- - // -----------Quote Token Part------------- - // we ignore the "spaces" TOKEN, wo include " ","\r","\n" and more to - // only be include in string TOKEN - if ident.Identifier == "" && !inQuote { - isSpaces = true - prevIndex = i - tempVal = sentence[prevIndex:i] - break - } - if ident.Identifier == DQUOTE && (QuoteIdentifier != "'" && !inQuoteStep) { - - // if the current lecture head is inside a string, we must be carefull about \", cause - // it does not end the current string. - // if we have a token DQUOTE without being in a string, its the start of a new string - if inQuote && QuoteIdentifier == "\"" { - if ret[len(ret)-1].Value[len(ret[len(ret)-1].Value)-1] != '\\' { - inQuote = false - QuoteIdentifier = "" - } - } else { - inQuote = true - inQuoteStep = true - QuoteIdentifier = "\"" - } - } - if ident.Identifier == SQUOTE && (QuoteIdentifier != "\"" && !inQuoteStep) { - - // if the current lecture head is inside a char, we must be carefull about \", cause - // it does not end the current string. - // if we have a token SQUOTE without being in a char, its the start of a new char - if inQuote && QuoteIdentifier == "'" { - if ret[len(ret)-1].Value[len(ret[len(ret)-1].Value)-1] != '\\' { - inQuote = false - QuoteIdentifier = "" - } - } else { - inQuote = true - inQuoteStep = true - QuoteIdentifier = "'" - } - } - // -----------Quote Token Part END------------- - - // -----------Special Token Part------------- - // - // we change our behavior in case of some special TOKEN, like ASSIGN, cause he can be a part of - // a complexe token (ADDASSIGN for exemple) - beforeChangeVal := tempVal - ret = tokenCommentGroup(ident, ret, &prevIndex, &tempVal, i) - ret = tokenComment(ident, ret, &prevIndex, &tempVal, i, line, sentence) - ret = tokenInt(ident, ret, &prevIndex, &tempVal, i, isSpaces) - ret = tokenPeriod(ident, ret, &prevIndex, &tempVal, i, isSpaces, inQuote, sentence) - ret = tokenAssign(ident, ret, &prevIndex, &tempVal, i, isSpaces) - ret = tokenDiv(ident, ret, &prevIndex, &tempVal, i) - ret = tokenAddSub(ident, ret, &prevIndex, &tempVal, i, isSpaces, ADD, INC) - ret = tokenAddSub(ident, ret, &prevIndex, &tempVal, i, isSpaces, SUB, DEC) - ret = tokenAddSub(ident, ret, &prevIndex, &tempVal, i, isSpaces, XORBIN, XOR) - if beforeChangeVal != tempVal { - break - } - // ---------Special Token Part END----------- - - // ---------Normal Token Part END----------- - // - // append a new Token to the variable ret - - if QuoteIdentifier == "\"" { - if inQuote { - if inQuote && tempVal == "\n" { - ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - QuoteIdentifier = "" - inQuote = false - } else { - ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - } - } else { - ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - QuoteIdentifier = "" - } - } else if QuoteIdentifier == "'" { - if inQuote { - if inQuote && tempVal == "\n" { - ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - QuoteIdentifier = "" - inQuote = false - } else { - ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - } - } else { - ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - QuoteIdentifier = "" - } - } else { - ret = inQuoteChange(ident.Identifier, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence) - } - - isSpaces = false - - tempVal = "" - prevIndex = i - // ---------Normal Token Part END----------- - } - // -----------Is Known Syntaxe Part END------------- - - // -----------Previous Token COMMENT Part Again------------- - // - // we must check again if the previous one is a COMMENT in case its a text, or the end of the lexing. - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == COMMENT && len(ret)-1 != endOfComm && !inQuote { - if tempVal == "/" && ret[len(ret)-1].Value == "" { - ret[len(ret)-1].TokenType = COMMENTGROUP - } else if tempVal != "\n" && tempVal != "\r" { - ret[len(ret)-1].Value += tempVal - } else { - endOfComm = len(ret) - 1 - } - canBeText = false - tempVal = "" - prevIndex = i - break - } - if ret[len(ret)-1].TokenType == COMMENTGROUP && len(ret)-1 != endOfCommGroup && !inQuote { - canBeText = false - if tempVal[len(tempVal)-1] == '/' { - break - } else if len(tempVal) > 1 { - if tempVal[len(tempVal)-2:] == "/#" { - endOfCommGroup = len(ret) - 1 - } else { - ret[len(ret)-1].Value += tempVal - } - } else { - ret[len(ret)-1].Value += tempVal - } - tempVal = "" - prevIndex = i - break - } - } - // -----------Previous Token COMMENT Part Again END------------- - } - // -----------Can still be text Part------------- - // - // if after checking all the known syntaxe, the tempValue can still - // be a TEXT, we parse the tempValue backward to verifies if - // a substring of tempValue can also be a known syntaxe - if canBeText { - for y := len(tempVal) - 1; y >= 0; y-- { - for _, ident := range Identifier { - if ident.Identifier != INT { - if ident.IsSyntaxe(tempVal[y:]) { - canBeText = false - ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, Identifier[0], tempVal[:y], prevIndex, sentence) - i += y - len(tempVal) - isSpaces = false - prevIndex = i - } - } - - } - } - } - // -----------Can still be text Part END------------- - } - // -----------End of lexer Part------------- - // - // if at the end of the sentence parse, tempVal is not "", it means that - // a last token of type TEXT must be appended to the return value - if len(ret) > 0 { - if ret[len(ret)-1].TokenType == COMMENTGROUP && tempVal == "/" { - ret[len(ret)-1].Value += tempVal - } else if tempVal != "" { - actualIndex, line = positionDetector(prevIndex, sentence) - ret = append(ret, addToken(Identifier[0].Identifier, tempVal, actualIndex, line)) - - prevIndex += len(tempVal) - } - } else if tempVal != "" { - actualIndex, line = positionDetector(prevIndex, sentence) - ret = append(ret, addToken(Identifier[0].Identifier, tempVal, actualIndex, line)) - - prevIndex += len(tempVal) - } - - // created a last token of type EOF (EndOfFile) - actualIndex, line = positionDetector(prevIndex, sentence) - ret = append(ret, addToken(Identifier[len(Identifier)-1].Identifier, "", actualIndex, line)) - - return ret - // -----------End of lexer Part END------------- -} - -func inQuoteChange(ttoken string, PreviousQuote string, inQuote bool, ret []Token, identi identifier, val string, prevIndex int, sentence string) []Token { - actualIndex, line := positionDetector(prevIndex, sentence) - if inQuote { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == ttoken { - ret[len(ret)-1].Value += val - } else { - if PreviousQuote == "'" { - ret = append(ret, addToken(CHAR, val, actualIndex, line)) - } else if PreviousQuote == "\"" { - ret = append(ret, addToken(STRING, val, actualIndex, line)) - } - } - } else { - if PreviousQuote == "'" { - ret = append(ret, addToken(CHAR, val, actualIndex, line)) - } else if PreviousQuote == "\"" { - ret = append(ret, addToken(STRING, val, actualIndex, line)) - } - } - } else { - ret = append(ret, addToken(identi.Identifier, val, actualIndex, line)) - } - return ret -} - -// positionDetector find the current position and line of the token we want to create. -// Take our current []Token, the index of lexing, and the global sentence. -// -// return position, line -func positionDetector(prevIndex int, sentence string) (int, int) { - var toRet = 0 - var line = 1 - for _, v := range sentence[:prevIndex] { - if v == '\n' { - toRet = 0 - line += 1 - } else { - toRet += 1 - } - } - return toRet + 1, line -} - -// addToken create a new token with the given parameters -// -// return the created token -func addToken(TokenType string, Value string, Position int, Line int) Token { - var ret Token - - ret.TokenType = TokenType - ret.Value = Value - ret.Position = Position - ret.Line = Line - - return ret -} - -// tokenAddSub replace the previous token.tokenType in ret to toReplace if the current token.tokenType is equal to toFind. -// -// return the changed []Token -func tokenAddSub(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int, isSpaces bool, toFind string, toReplace string) []Token { - if ident.Identifier == ADD || ident.Identifier == SUB || ident.Identifier == XORBIN { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == toFind && ret[len(ret)-1].TokenType == ident.Identifier && !isSpaces { - ret[len(ret)-1].TokenType = toReplace - ret[len(ret)-1].Value += *tempVal - *tempVal = "" - *prevIndex = index - } - } - } - return ret -} - -// tokenAssign replace the previous token.tokenType in ret to the compose token of the current token and the previous one -// if the current token.tokenType is ASSIGN, and if the previous one is one of the listed token able to merge with an -// ASSIGN. -// -// return the changed []Token -func tokenAssign(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int, isSpaces bool) []Token { - if ident.Identifier == ASSIGN && !isSpaces { - if len(ret) >= 1 { - if concatEqual(ret[len(ret)-1].TokenType) { - if ret[len(ret)-1].TokenType == ASSIGN { - ret[len(ret)-1].TokenType = EQUAL - } else if ret[len(ret)-1].TokenType == ADD || ret[len(ret)-1].TokenType == SUB || ret[len(ret)-1].TokenType == MULT || ret[len(ret)-1].TokenType == DIV || ret[len(ret)-1].TokenType == QOT || ret[len(ret)-1].TokenType == MOD { - ret[len(ret)-1].TokenType = ret[len(ret)-1].TokenType + ident.Identifier - } else if ret[len(ret)-1].TokenType == LSS { - ret[len(ret)-1].TokenType = LEQ - } else if ret[len(ret)-1].TokenType == GTR { - ret[len(ret)-1].TokenType = GEQ - } else if ret[len(ret)-1].TokenType == NOT { - ret[len(ret)-1].TokenType = NEQ - } - ret[len(ret)-1].Value += *tempVal - *tempVal = "" - *prevIndex = index - } - } - } - return ret -} - -// tokenDiv replace the previous token.tokenType in ret to the QOT token -// if the current token.tokenType and the previous token.tokenType are DIV -// -// return the changed []Token -func tokenDiv(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int) []Token { - if ident.Identifier == DIV { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == DIV { - ret[len(ret)-1].TokenType = QOT - ret[len(ret)-1].Value += *tempVal - *tempVal = "" - *prevIndex = index - } - } - } - return ret -} - -// TokenPeriod replace the previous token.tokenType in ret to the FLOAT token -// if the current token.tokenType is PERIOD and the previous token.tokenType is INT -// -// if we are in a string, ignore this behavior -// -// return the changed []Token -func tokenPeriod(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int, isSpaces bool, inQuote bool, sentence string) []Token { - if ident.Identifier == PERIOD { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == INT && !isSpaces { - ret[len(ret)-1].Value += *tempVal - ret[len(ret)-1].TokenType = FLOAT - *tempVal = "" - *prevIndex = index - } else if !inQuote { - actualIndex, line := positionDetector(*prevIndex, sentence) - ret = append(ret, addToken(ident.Identifier, *tempVal, actualIndex, line)) - *tempVal = "" - *prevIndex = index - } - } - } - return ret -} - -// tokenDiv replace the previous token.tokenType in ret to the QOT tokenType -// if the current token.tokenType and the previous token.tokenType are DIV -// -// return the changed []Token -func tokenInt(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int, isSpaces bool) []Token { - if ident.Identifier == INT { - if !isSpaces { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == INT || ret[len(ret)-1].TokenType == TEXT || ret[len(ret)-1].TokenType == FLOAT { - ret[len(ret)-1].Value += *tempVal - *tempVal = "" - *prevIndex = index - } - } - } - - } - return ret -} - -// tokenComment replace the previous token.tokenType in ret to the COMMENTGROUP tokenType -// if the current token.tokenType is COMMENTGROUPIDENT, and of the previous one is COMMENT -// -// return the changed []Token -func tokenCommentGroup(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int) []Token { - if ident.Identifier == COMMENTGROUPIDENT { - if len(ret) >= 1 { - if ret[len(ret)-1].TokenType == COMMENT { - ret[len(ret)-1].TokenType = COMMENTGROUP - ret[len(ret)-1].Value += "" - *tempVal = "" - *prevIndex = index - } - } - } - return ret -} - -// tokenComment add a new token if the no value if the identifier is COMMENT -// -// return the changed []Token -func tokenComment(ident identifier, ret []Token, prevIndex *int, tempVal *string, index int, line int, sentence string) []Token { - if ident.Identifier == COMMENT { - actualIndex, line := positionDetector(*prevIndex, sentence) - ret = append(ret, addToken(ident.Identifier, "", actualIndex, line)) - *tempVal = "" - *prevIndex = index - } - return ret -} diff --git a/lexer/lexerIdentifier.go b/lexer/lexerIdentifier.go deleted file mode 100644 index a221a17..0000000 --- a/lexer/lexerIdentifier.go +++ /dev/null @@ -1,301 +0,0 @@ -package lexer - -// each type of token -var ( - TEXT = "TEXT" - STRING = "STRING" - CHAR = "CHAR" - PRINT = "PRINT" - INT = "INT" - FLOAT = "FLOAT" - ADD = "ADD" - SUB = "SUB" - MULT = "MULT" - DIV = "DIV" - QOT = "QOT" - MOD = "MOD" - INC = "INC" - DEC = "DEC" - ASSIGN = "ASSIGN" - LSS = "LSS" - GTR = "GTR" - NEQ = "NEQ" - LEQ = "LEQ" - GEQ = "GEQ" - XOR = "XOR" - XORBIN = "XORBIN" - OR = "OR" - AND = "AND" - EQUAL = "EQUAL" - LPAREN = "LPAREN" - RPAREN = "RPAREN" - EOL = "EOL" - DQUOTE = "DQUOTE" - SQUOTE = "SQUOTE" - PERIOD = "PERIOD" - COLON = "COLON" - LBRACE = "LBRACE" - RBRACE = "RBRACE" - LBRACKET = "LBRACKET" - RBRACKET = "RBRACKET" - COMMA = "COMMA" - NOT = "NOT" - BOOL = "BOOL" - MURLOC = "MURLOC" - EOF = "EOF" - COMMENT = "COMMENT" - COMMENTGROUP = "COMMENTGROUP" - COMMENTGROUPIDENT = "COMMENTGROUPIDENT" -) - -//--------------------------------------------// -// Need to be optimized -// Change for a switch case of a hashmap -//--------------------------------------------// - -// link between syntax and token -var Identifier []identifier = []identifier{ - { - Identifier: TEXT, - Syntax: []string{}, - }, - { - Identifier: INT, - Syntax: []string{ - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - }, - }, - { - Identifier: ADD, - Syntax: []string{ - "+", - }, - }, - { - Identifier: SUB, - Syntax: []string{ - "-", - }, - }, - { - Identifier: MULT, - Syntax: []string{ - "*", - }, - }, - { - Identifier: DIV, - Syntax: []string{ - "/", - }, - }, - { - Identifier: MOD, - Syntax: []string{ - "%", - }, - }, - { - Identifier: ASSIGN, - Syntax: []string{ - "=", - }, - }, - { - Identifier: GTR, - Syntax: []string{ - ">", - }, - }, - { - Identifier: LSS, - Syntax: []string{ - "<", - }, - }, - { - Identifier: XORBIN, - Syntax: []string{ - "^", - }, - }, - { - Identifier: AND, - Syntax: []string{ - "&&", - }, - }, - { - Identifier: OR, - Syntax: []string{ - "||", - }, - }, - { - Identifier: LPAREN, - Syntax: []string{ - "(", - }, - }, - { - Identifier: RPAREN, - Syntax: []string{ - ")", - }, - }, - { - Identifier: EOL, - Syntax: []string{ - ";", - }, - }, - { - Identifier: DQUOTE, - Syntax: []string{ - "\"", - }, - }, - { - Identifier: SQUOTE, - Syntax: []string{ - "'", - }, - }, - { - Identifier: MURLOC, - Syntax: []string{ - "mgrlgrl", - }, - }, - { - Identifier: PERIOD, - Syntax: []string{ - ".", - }, - }, - { - Identifier: COLON, - Syntax: []string{ - ":", - }, - }, - { - Identifier: COMMA, - Syntax: []string{ - ",", - }, - }, - { - Identifier: LBRACE, - Syntax: []string{ - "{", - }, - }, - { - Identifier: RBRACE, - Syntax: []string{ - "}", - }, - }, - { - Identifier: LBRACKET, - Syntax: []string{ - "[", - }, - }, - { - Identifier: RBRACKET, - Syntax: []string{ - "]", - }, - }, - { - Identifier: COMMENT, - Syntax: []string{ - "#", - }, - }, - { - Identifier: "", - Syntax: []string{ - " ", - "\n", - "\t", - "\r", - }, - }, - { - Identifier: BOOL, - Syntax: []string{ - "true", - "false", - }, - }, - { - Identifier: NOT, - Syntax: []string{ - "!", - }, - }, - { - Identifier: EOF, - Syntax: []string{}, - }, -} - -type identifier struct { - Identifier string - Syntax []string -} - -// IsSyntaxe verify is the string tempVal exist in the current identifier -// -// return a true false value -func (ident identifier) IsSyntaxe(tempVal string) bool { - for _, syntaxe := range ident.Syntax { - if syntaxe == tempVal { - return true - } - } - return false -} - -// concatEqual verify is the string str is equal with one of the -// mathematical opperand -// -// return a true false value -func concatEqual(str string) bool { - switch str { - case ADD: - return true - case MOD: - return true - case SUB: - return true - case QOT: - return true - case MULT: - return true - case ASSIGN: - return true - case DIV: - return true - case GTR: - return true - case LSS: - return true - case NOT: - return true - default: - return false - } -} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index fffaadb..a638bc9 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -13,7 +13,7 @@ func tLexer(t *testing.T, tested testList, name string) { result += "\n--------------------------------------------------\n--------------------------------------------------\n\t\t---" + name + "-INPUT---\n--------------------------------------------------\n" + code + "\n--------------------------------------------------" result += "\n\t\t---DIFF LIST---\n--------------------------------------------------\n" diff := 0 - l := Lexer(code) + l := LexerR(code) if l == nil { result += "Expected a lexer, got nil\n--------------------------------------------------\n" } else if len(l) != expectedLenth { @@ -96,720 +96,3 @@ func TestCharString2(t *testing.T) { func TestMultiLigneString(t *testing.T) { tLexer(t, testMultiLigneString, "testMultiLigneString") } - -func TestPositionDetector_1(t *testing.T) { - var Expected1 = 1 //position - var Expected2 = 1 //line - - var sentence = "test" - var previndex = 0 - - var got1, got2 = positionDetector(previndex, sentence) - - var errorString = "\n" - - if got1 != Expected1 { - errorString += "Failed at getting Position\n expected\t: " + strconv.Itoa(Expected1) + "\n got \t\t: " + strconv.Itoa(got1) + "\n" - } - if got2 != Expected2 { - errorString += "Failed at getting Line\n expected\t: " + strconv.Itoa(Expected2) + "\n got \t\t: " + strconv.Itoa(got2) + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestPositionDetector_2(t *testing.T) { - var Expected1 = 1 //position - var Expected2 = 2 //line - - var sentence = "test\ntest" - var previndex = 5 - - var got1, got2 = positionDetector(previndex, sentence) - - var errorString = "\n" - - if got1 != Expected1 { - errorString += "Failed at getting Position\n expected\t: " + strconv.Itoa(Expected1) + "\n got \t\t: " + strconv.Itoa(got1) + "\n" - } - if got2 != Expected2 { - errorString += "Failed at getting Line\n expected\t: " + strconv.Itoa(Expected2) + "\n got \t\t: " + strconv.Itoa(got2) + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestPositionDetector_3(t *testing.T) { - var Expected1 = 3 //position - var Expected2 = 1 //line - - var sentence = "test\ntest" - var previndex = 2 - - var got1, got2 = positionDetector(previndex, sentence) - - var errorString = "\n" - - if got1 != Expected1 { - errorString += "Failed at getting Position\n expected\t: " + strconv.Itoa(Expected1) + "\n got \t\t: " + strconv.Itoa(got1) + "\n" - } - if got2 != Expected2 { - errorString += "Failed at getting Line\n expected\t: " + strconv.Itoa(Expected2) + "\n got \t\t: " + strconv.Itoa(got2) + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestAddToken(t *testing.T) { - var Expected Token = Token{ - TokenType: INT, - Value: "42", - Position: 4, - Line: 2, - } - - var got = addToken(INT, "42", 4, 2) - - if got != Expected { - t.Error("Failed at getting Position\n expected\t: ", Expected, "\n got \t\t: ", got, "\n") - } - -} - -func TestTokenAddSub_1(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: INC, - Value: "++", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: ADD, - Value: "+", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "+" - var ident = identifier{"ADD", []string{}} - var got = tokenAddSub(ident, ret, &prevIndex, &tempVal, 2, false, ADD, INC) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to INC\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestTokenAddSub_2(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: TEXT, - Value: "pi", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: TEXT, - Value: "pi", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "+" - var ident = identifier{"ADD", []string{}} - var got = tokenAddSub(ident, ret, &prevIndex, &tempVal, 2, false, ADD, INC) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to INC\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 1 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "+" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestTokenAddSub_3(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: DEC, - Value: "--", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: SUB, - Value: "-", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "-" - var ident = identifier{"SUB", []string{}} - var got = tokenAddSub(ident, ret, &prevIndex, &tempVal, 2, false, SUB, DEC) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to DEC\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestTokenAssign_EQUAL(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: EQUAL, - Value: "==", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: ASSIGN, - Value: "=", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to DEC\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestTokenAssign_LEQ(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: LEQ, - Value: "<=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: LSS, - Value: "<", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to DEC\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } -} - -func TestTokenAssign_GEQ(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: GEQ, - Value: ">=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: GTR, - Value: ">", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to GEQ\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_ADDASSIGN(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: ADD + ASSIGN, - Value: "+=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: ADD, - Value: "+", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to ADDASSIGN\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_SUBASSIGN(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: SUB + ASSIGN, - Value: "-=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: SUB, - Value: "-", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to SUBASSIGN\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_MULTASSIGN(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: MULT + ASSIGN, - Value: "*=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: MULT, - Value: "*", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to MULTASSIGN\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_DIVASSIGN(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: DIV + ASSIGN, - Value: "/=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: DIV, - Value: "/", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to DIVASSIGN\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_MODASSIGN(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: MOD + ASSIGN, - Value: "%=", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: MOD, - Value: "%", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "=" - - var got = tokenAssign(identifier{Identifier: ASSIGN, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at changing to MODASSIGN\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenAssign_BAD(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: MOD, - Value: "%", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: MOD, - Value: "%", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "+" - - var got = tokenAssign(identifier{Identifier: ADD, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 1 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "+" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenDiv_OK(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: QOT, - Value: "//", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: DIV, - Value: "/", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "/" - - var got = tokenDiv(identifier{Identifier: DIV, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at modifying to MOD\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestTokenDiv_BAD(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: DIV, - Value: "/", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: DIV, - Value: "/", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "+" - - var got = tokenDiv(identifier{Identifier: ADD, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2) - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 1 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(1) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "+" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestPeriod_FLOAT(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: FLOAT, - Value: "1.", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: INT, - Value: "1", - Position: 1, - Line: 1, - }, - } - var prevIndex = 1 - var tempVal = "." - - var got = tokenPeriod(identifier{Identifier: PERIOD, Syntax: []string{}}, ret, &prevIndex, &tempVal, 2, false, false, "1.") - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed at modifying to FLOAT\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 2 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} - -func TestPeriod_BAD(t *testing.T) { - var Expected = []Token{ - Token{ - TokenType: INT, - Value: "1", - Position: 1, - Line: 1, - }, - } - - var ret = []Token{ - Token{ - TokenType: INT, - Value: "1", - Position: 1, - Line: 1, - }, - } - var prevIndex = 2 - var tempVal = "." - - var got = tokenPeriod(identifier{Identifier: PERIOD, Syntax: []string{}}, ret, &prevIndex, &tempVal, 3, true, false, "1 .") - - var errorString = "\n" - - if got[0] != Expected[0] { - errorString += "Failed\n expected\t: " + Expected[0].TokenType + "\n got \t\t: " + got[0].TokenType + "\n" - } - if prevIndex != 3 { - errorString += "Failed at modifying pos\n expected\t: " + strconv.Itoa(2) + "\n got \t\t: " + strconv.Itoa(prevIndex) + "\n" - } - if tempVal != "" { - errorString += "Failed at modifying val\n expected\t: " + "\n got \t\t: " + tempVal + "\n" - } - if len(errorString) != 1 { - t.Error(errorString) - } - -} diff --git a/lexer/lexer_test_list.go b/lexer/lexer_test_list.go index 5ad79ff..e1cb21a 100644 --- a/lexer/lexer_test_list.go +++ b/lexer/lexer_test_list.go @@ -400,10 +400,6 @@ var ( }, }, } - testCHARSTRING3 = testList{ - input: `'\'' and '\"' and "\"" and "\'"`, - output: []Token{}, - } testEOL = testList{ input: "();\nmgrlgrl;\n_aa_", output: []Token{ @@ -707,7 +703,7 @@ var ( }, { TokenType: STRING, - Value: "hello\n", + Value: "hello", Position: 7, Line: 1, }, diff --git a/lexer/v2/lexer_test.go b/lexer/v2/lexer_test.go deleted file mode 100644 index 0cd3d89..0000000 --- a/lexer/v2/lexer_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package v2 - -import ( - "strconv" - "testing" -) - -func tLexer(t *testing.T, tested testList, name string) { - result := "" - code := tested.input - expected := tested.output - expectedLenth := len(expected) - result += "\n--------------------------------------------------\n--------------------------------------------------\n\t\t---" + name + "-INPUT---\n--------------------------------------------------\n" + code + "\n--------------------------------------------------" - result += "\n\t\t---DIFF LIST---\n--------------------------------------------------\n" - diff := 0 - l := LexerR(code) - if l == nil { - result += "Expected a lexer, got nil\n--------------------------------------------------\n" - } else if len(l) != expectedLenth { - result += "Expected " + strconv.Itoa(expectedLenth) + " tokens, got " + strconv.Itoa(len(l)) - diff++ - } - for Position, expct := range expected { - if Position < len(l) { - if expct != l[Position] { - diff++ - result += "\n--------------------------------------------------\nDiff " + strconv.Itoa(diff) + " Expected {" + expct.TokenType + " " + expct.Value + " " + strconv.Itoa(expct.Position) + " " + strconv.Itoa(expct.Line) + "} for the token n°" + strconv.Itoa(Position+1) + "\n Got \t{" + l[Position].TokenType + " " + l[Position].Value + " " + strconv.Itoa(l[Position].Position) + " " + strconv.Itoa(l[Position].Line) + "}\n--------------------------------------------------\n" - } - } - } - if diff == 0 { - result += "\t ---AUCUNE ERREUR---\n--------------------------------------------------\n" - t.Log(result) - } else { - result += "\n--------------------------------------------------\n" - t.Error(result, "\ngot :\n", l) - } -} - -//func TestHashtag5(t *testing.T) { -// tLexer(t, testHashtag5, " testHashtag5") -//} - -func TestNoFile(t *testing.T) { - tLexer(t, testNoFile, " testNoFile") -} - -func TestCalc(t *testing.T) { - tLexer(t, testCalc, " testCalc") -} - -func TestBoolOpperand(t *testing.T) { - tLexer(t, testBoolOpperand, " testBoolOpperand") -} - -func TestEOL(t *testing.T) { - tLexer(t, testEOL, "testEOL") -} - -func TestHashtag(t *testing.T) { - tLexer(t, testHashtag, " testHashtag") -} - -func TestHashtag2(t *testing.T) { - tLexer(t, testHashtag2, " testHashtag2") -} - -func TestHashtag3(t *testing.T) { - tLexer(t, testHashtag3, " testHashtag3") -} - -func TestHashtag4(t *testing.T) { - tLexer(t, testHashtag4, " testHashtag4") -} - -func TestMurloc(t *testing.T) { - tLexer(t, testMurloc, " testMurloc") -} - -func TestSpeChar(t *testing.T) { - tLexer(t, testSpeChar, " testSpeChar") -} - -func TestChar(t *testing.T) { - tLexer(t, testCHAR, "testCHar") -} - -func TestCharString(t *testing.T) { - tLexer(t, testCHARSTRING, "testCharString") -} - -func TestCharString2(t *testing.T) { - tLexer(t, testCHARSTRING2, "testCharString2") -} diff --git a/lexer/v2/lexer_test_list.go b/lexer/v2/lexer_test_list.go deleted file mode 100644 index 1846643..0000000 --- a/lexer/v2/lexer_test_list.go +++ /dev/null @@ -1,677 +0,0 @@ -package v2 - -type testList struct { - input string - output []Token -} - -var ( - testCalc = testList{ - input: `+= -= /= *= %= ++ -- a++ b-- //= ^^ ^`, - output: []Token{ - { - TokenType: ADD + ASSIGN, - Value: `+=`, - Position: 1, - Line: 1, - }, - { - TokenType: SUB + ASSIGN, - Value: `-=`, - Position: 4, - Line: 1, - }, - { - TokenType: DIV + ASSIGN, - Value: `/=`, - Position: 7, - Line: 1, - }, - { - TokenType: MULT + ASSIGN, - Value: `*=`, - Position: 10, - Line: 1, - }, - { - TokenType: MOD + ASSIGN, - Value: `%=`, - Position: 13, - Line: 1, - }, - { - TokenType: INC, - Value: `++`, - Position: 16, - Line: 1, - }, - { - TokenType: DEC, - Value: `--`, - Position: 19, - Line: 1, - }, - { - TokenType: TEXT, - Value: `a`, - Position: 22, - Line: 1, - }, - { - TokenType: INC, - Value: `++`, - Position: 23, - Line: 1, - }, - { - TokenType: TEXT, - Value: `b`, - Position: 26, - Line: 1, - }, - { - TokenType: DEC, - Value: `--`, - Position: 27, - Line: 1, - }, - { - TokenType: QOT + ASSIGN, - Value: `//=`, - Position: 30, - Line: 1, - }, - { - TokenType: XOR, - Value: `^^`, - Position: 34, - Line: 1, - }, - { - TokenType: XORBIN, - Value: `^`, - Position: 37, - Line: 1, - }, - { - TokenType: EOF, - Value: "", - Position: 38, - Line: 1, - }, - }, - } - testDQuote = testList{ - input: `" "a"be"`, - output: []Token{ - { - TokenType: DQUOTE, - Value: `"`, - Position: 1, - Line: 1, - }, - { - TokenType: STRING, - Value: ` `, - Position: 2, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 6, - Line: 1, - }, - { - TokenType: TEXT, - Value: `a`, - Position: 7, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 8, - Line: 1, - }, - { - TokenType: STRING, - Value: `be`, - Position: 9, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 11, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 12, - Line: 1, - }, - }, - } - testMurloc = testList{ - input: `shrek is love, mgrlgrl is life`, - output: []Token{ - { - TokenType: TEXT, - Value: `shrek`, - Position: 1, - Line: 1, - }, - { - TokenType: TEXT, - Value: `is`, - Position: 7, - Line: 1, - }, - { - TokenType: TEXT, - Value: `love`, - Position: 10, - Line: 1, - }, - { - TokenType: COMMA, - Value: `,`, - Position: 14, - Line: 1, - }, - { - TokenType: MURLOC, - Value: `mgrlgrl`, - Position: 16, - Line: 1, - }, - { - TokenType: TEXT, - Value: `is`, - Position: 24, - Line: 1, - }, - { - TokenType: TEXT, - Value: `life`, - Position: 27, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 31, - Line: 1, - }, - }, - } - testSpeChar = testList{ - input: ":;\n.,()[]{}", - output: []Token{ - { - TokenType: COLON, - Value: `:`, - Position: 1, - Line: 1, - }, - { - TokenType: EOL, - Value: `;`, - Position: 2, - Line: 1, - }, - { - TokenType: PERIOD, - Value: `.`, - Position: 1, - Line: 2, - }, - { - TokenType: COMMA, - Value: `,`, - Position: 2, - Line: 2, - }, - { - TokenType: LPAREN, - Value: `(`, - Position: 3, - Line: 2, - }, - { - TokenType: RPAREN, - Value: `)`, - Position: 4, - Line: 2, - }, - { - TokenType: LBRACKET, - Value: `[`, - Position: 5, - Line: 2, - }, - { - TokenType: RBRACKET, - Value: `]`, - Position: 6, - Line: 2, - }, - { - TokenType: LBRACE, - Value: `{`, - Position: 7, - Line: 2, - }, - { - TokenType: RBRACE, - Value: `}`, - Position: 8, - Line: 2, - }, - { - TokenType: EOF, - Value: ``, - Position: 9, - Line: 2, - }}, - } - testCHAR = testList{ - input: "'Ok'", - output: []Token{ - { - TokenType: SQUOTE, - Value: `'`, - Position: 1, - Line: 1, - }, - { - TokenType: CHAR, - Value: `Ok`, - Position: 2, - Line: 1, - }, - { - TokenType: SQUOTE, - Value: `'`, - Position: 4, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 5, - Line: 1, - }, - }, - } - testCHARSTRING = testList{ - input: "\"Ok'\"\"'Ok\"", - output: []Token{ - { - TokenType: DQUOTE, - Value: `"`, - Position: 1, - Line: 1, - }, - { - TokenType: STRING, - Value: `Ok'`, - Position: 2, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 5, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 6, - Line: 1, - }, - { - TokenType: STRING, - Value: `'Ok`, - Position: 7, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 10, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 11, - Line: 1, - }, - }, - } - testCHARSTRING2 = testList{ - input: `'Ok'"'Ok"`, - output: []Token{ - { - TokenType: SQUOTE, - Value: `'`, - Position: 1, - Line: 1, - }, - { - TokenType: CHAR, - Value: `Ok`, - Position: 2, - Line: 1, - }, - { - TokenType: SQUOTE, - Value: `'`, - Position: 4, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 5, - Line: 1, - }, - { - TokenType: STRING, - Value: `'Ok`, - Position: 6, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 9, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 10, - Line: 1, - }, - }, - } - testEOL = testList{ - input: "();\nmgrlgrl;\n_aa_", - output: []Token{ - { - TokenType: LPAREN, - Value: `(`, - Position: 1, - Line: 1, - }, - { - TokenType: RPAREN, - Value: `)`, - Position: 2, - Line: 1, - }, - { - TokenType: EOL, - Value: `;`, - Position: 3, - Line: 1, - }, - { - TokenType: MURLOC, - Value: `mgrlgrl`, - Position: 1, - Line: 2, - }, - { - TokenType: EOL, - Value: `;`, - Position: 8, - Line: 2, - }, - { - TokenType: TEXT, - Value: `_aa_`, - Position: 1, - Line: 3, - }, - { - TokenType: EOF, - Value: ``, - Position: 5, - Line: 3, - }, - }, - } - testNoFile = testList{ - input: "", - output: []Token{ - { - TokenType: EOF, - Value: ``, - Position: 1, - Line: 1, - }, - }, - } - testHashtag = testList{ - input: "prout# in comment\n#/ in commentgroup\n and next ligne\n and test for / and #/ /#\nOutside of the comment group", - output: []Token{ - { - TokenType: TEXT, - Value: "prout", - Position: 1, - Line: 1, - }, - { - TokenType: COMMENT, - Value: " in comment", - Position: 6, - Line: 1, - }, - { - TokenType: COMMENTGROUP, - Value: " in commentgroup\n and next ligne\n and test for / and #/ ", - Position: 1, - Line: 2, - }, - { - TokenType: TEXT, - Value: `Outside`, - Position: 1, - Line: 5, - }, - { - TokenType: TEXT, - Value: `of`, - Position: 9, - Line: 5, - }, - { - TokenType: TEXT, - Value: `the`, - Position: 12, - Line: 5, - }, - { - TokenType: TEXT, - Value: `comment`, - Position: 16, - Line: 5, - }, - { - TokenType: TEXT, - Value: `group`, - Position: 24, - Line: 5, - }, - { - TokenType: EOF, - Value: ``, - Position: 29, - Line: 5, - }, - }, - } - testHashtag2 = testList{ - input: "prout# in comment\n#/ in commentgroup\n and next ligne\n and test for / and #/", - output: []Token{ - { - TokenType: TEXT, - Value: `prout`, - Position: 1, - Line: 1, - }, - { - TokenType: COMMENT, - Value: ` in comment`, - Position: 6, - Line: 1, - }, - { - TokenType: COMMENTGROUP, - Value: " in commentgroup\n and next ligne\n and test for / and #/", - Position: 1, - Line: 2, - }, - { - TokenType: EOF, - Value: ``, - Position: 22, - Line: 4, - }, - }, - } - testHashtag3 = testList{ - input: "\"#prout\"", - output: []Token{ - { - TokenType: DQUOTE, - Value: `"`, - Position: 1, - Line: 1, - }, - { - TokenType: STRING, - Value: `#prout`, - Position: 2, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 8, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 9, - Line: 1, - }, - }, - } - testHashtag4 = testList{ - input: "\"#/prout/#\"", - output: []Token{ - { - TokenType: DQUOTE, - Value: `"`, - Position: 1, - Line: 1, - }, - { - TokenType: STRING, - Value: `#/prout/#`, - Position: 2, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 11, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 12, - Line: 1, - }, - }, - } - testHashtag5 = testList{ - input: "#ok\nimport console;\n\n\n#test Print\nconsole.println(\"Test Print\");\nconsole.println(\"Hello World\");\nconsole.println(\"Test Print Past\");", - output: []Token{ - { - TokenType: COMMENTGROUP, - Value: `#prout`, - Position: 1, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 11, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 12, - Line: 1, - }, - }, - } - testBoolOpperand = testList{ - input: "&& || ^ \"&& || ^\"", - output: []Token{ - { - TokenType: AND, - Value: `&&`, - Position: 1, - Line: 1, - }, - { - TokenType: OR, - Value: `||`, - Position: 4, - Line: 1, - }, - { - TokenType: XORBIN, - Value: `^`, - Position: 7, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 9, - Line: 1, - }, - { - TokenType: STRING, - Value: `&& || ^`, - Position: 10, - Line: 1, - }, - { - TokenType: DQUOTE, - Value: `"`, - Position: 17, - Line: 1, - }, - { - TokenType: EOF, - Value: ``, - Position: 18, - Line: 1, - }, - }, - } -) diff --git a/lexer/v2/main/main.go b/lexer/v2/main/main.go deleted file mode 100644 index dc4586e..0000000 --- a/lexer/v2/main/main.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import v2 "github.com/Eclalang/Ecla/lexer/v2" - -func main() { - tokens := v2.LexerR("/#ok#/a /##/") - for i, token := range tokens { - printToken(i, token) - } -} - -func printToken(i int, token v2.Token) { - println("\n-------Token", i, "-------") - println("Tokentype: ", token.TokenType) - println("Value: ", token.Value) - println("Line: ", token.Line) - println("Position: ", token.Position) -}