Merge pull request #287 from Eclalang/issue-#242-multiligne-str

fix multi ligne string for lexer. PTGLK !
Eclalang · Feb 27, 2024 · 5d8ccab · 5d8ccab
2 parents 77ef8a1 + 0b33462
commit 5d8ccab
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 7 deletions.
diff --git a/lexer/lexer.go b/lexer/lexer.go
@@ -50,6 +50,7 @@ func Lexer(sentence string) []Token {
 			// syntaxes with our tempVal, If the comparison is true,
 			// tempVal is a known syntaxes, and then a token
 			if ident.IsSyntaxe(tempVal) {
+
 				// canot be a text now
 				canBeText = false
 				if (ident.Identifier == COMMENT || ident.Identifier == COMMENTGROUP) && inQuote && len(ret) != 0 {
@@ -134,8 +135,6 @@ func Lexer(sentence string) []Token {
 						inQuoteStep = true
 						QuoteIdentifier = "'"
 					}
-				} else {
-
 				}
 				// -----------Quote Token Part END-------------
 
@@ -164,14 +163,26 @@ func Lexer(sentence string) []Token {
 
 				if QuoteIdentifier == "\"" {
 					if inQuote {
-						ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+						if inQuote && tempVal == "\n" {
+							ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+							QuoteIdentifier = ""
+							inQuote = false
+						} else {
+							ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+						}
 					} else {
 						ret = inQuoteChange(STRING, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
 						QuoteIdentifier = ""
 					}
 				} else if QuoteIdentifier == "'" {
 					if inQuote {
-						ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+						if inQuote && tempVal == "\n" {
+							ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+							QuoteIdentifier = ""
+							inQuote = false
+						} else {
+							ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
+						}
 					} else {
 						ret = inQuoteChange(CHAR, QuoteIdentifier, inQuote && !inQuoteStep, ret, ident, tempVal, prevIndex, sentence)
 						QuoteIdentifier = ""

diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -93,9 +93,9 @@ func TestCharString2(t *testing.T) {
 	tLexer(t, testCHARSTRING2, "testCharString2")
 }
 
-/*func TestCharString3(t *testing.T) {
-	tLexer(t, testCHARSTRING3, "testCharString3")
-}*/
+func TestMultiLigneString(t *testing.T) {
+	tLexer(t, testMultiLigneString, "testMultiLigneString")
+}
 
 func TestPositionDetector_1(t *testing.T) {
 	var Expected1 = 1 //position

diff --git a/lexer/lexer_test_list.go b/lexer/lexer_test_list.go
@@ -678,4 +678,63 @@ var (
 			},
 		},
 	}
+	testMultiLigneString = testList{
+		input: "b := \"hello\nworld\";",
+		output: []Token{
+			{
+				TokenType: TEXT,
+				Value:     `b`,
+				Position:  1,
+				Line:      1,
+			},
+			{
+				TokenType: COLON,
+				Value:     `:`,
+				Position:  3,
+				Line:      1,
+			},
+			{
+				TokenType: ASSIGN,
+				Value:     `=`,
+				Position:  4,
+				Line:      1,
+			},
+			{
+				TokenType: DQUOTE,
+				Value:     `"`,
+				Position:  6,
+				Line:      1,
+			},
+			{
+				TokenType: STRING,
+				Value:     "hello\n",
+				Position:  7,
+				Line:      1,
+			},
+			{
+				TokenType: TEXT,
+				Value:     `world`,
+				Position:  1,
+				Line:      2,
+			},
+			{
+				TokenType: DQUOTE,
+				Value:     `"`,
+				Position:  6,
+				Line:      2,
+			},
+			{
+				TokenType: STRING,
+				Value:     `;`,
+				Position:  7,
+				Line:      2,
+			},
+			{
+				TokenType: EOF,
+				Value:     ``,
+				Position:  8,
+				Line:      2,
+			},
+		},
+	}
 )