handle multiline comments

ast/ast.go

@@ -517,6 +517,12 @@ func (t *SQLToken) String() string {
 	case *token.SQLWord:
 		return v.String()
 	case string:
+		if t.Kind == token.Comment {
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
+		}
 		return v
 	default:
 		return " "
@@ -529,7 +535,10 @@ func (t *SQLToken) NoQuateString() string {
 		return v.NoQuateString()
 	case string:
 		if t.Kind == token.Comment {
-			return "--" + v + "\n"
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
 		}
 		return v
 	default:
@@ -543,7 +552,10 @@ func (t *SQLToken) Render(opts *RenderOptions) string {
 		return renderSQLWord(v, opts)
 	case string:
 		if t.Kind == token.Comment {
-			return "--" + v + "\n"
+			return "--" + v
+		}
+		if t.Kind == token.MultilineComment {
+			return "/*" + v + "*/"
 		}
 		return v
 	default:

internal/formatter/formatter.go

@@ -235,6 +235,15 @@ func formatItem(node ast.Node, env *formatEnvironment) ast.Node {
 		results = append(results, linebreakNode)
 		results = append(results, env.genIndent()...)
 	}
+	commentAfterMatcher := astutil.NodeMatcher{
+		ExpectTokens: []token.Kind{
+			token.Comment,
+		},
+	}
+	if commentAfterMatcher.IsMatch(node) {
+		results = append(results, linebreakNode)
+	}
+
 	breakStatementAfterMatcher := astutil.NodeMatcher{
 		ExpectTokens: []token.Kind{
 			token.Semicolon,

parser/parser.go

@@ -526,6 +526,12 @@ func parseAliased(reader *astutil.NodeReader) ast.Node {
 	}
 }
 
+var commentInfixMatcher = astutil.NodeMatcher{
+	ExpectTokens: []token.Kind{
+		token.Comment,
+		token.MultilineComment,
+	},
+}
 var identifierListInfixMatcher = astutil.NodeMatcher{
 	ExpectTokens: []token.Kind{
 		token.Comma,
@@ -567,7 +573,7 @@ func parseIdentifierList(reader *astutil.NodeReader) ast.Node {
 		peekNode            ast.Node
 	)
 	for {
-		if !tmpReader.PeekNodeIs(true, identifierListTargetMatcher) {
+		if !tmpReader.PeekNodeIs(true, identifierListTargetMatcher) && !tmpReader.PeekNodeIs(true, commentInfixMatcher) {
 			// Include white space after the comma
 			peekIndex, peekNode := tmpReader.PeekNode(true)
 			if peekNode != nil {
@@ -581,12 +587,18 @@ func parseIdentifierList(reader *astutil.NodeReader) ast.Node {
 			}
 			break
 		}
+		for tmpReader.PeekNodeIs(true, commentInfixMatcher) {
+			tmpReader.NextNode(true)
+		}
 
 		peekIndex, peekNode = tmpReader.PeekNode(true)
 		idents = append(idents, peekNode)
 		endIndex = peekIndex
 
 		tmpReader.NextNode(true)
+		//for tmpReader.PeekNodeIs(true, commentInfixMatcher) {
+		//tmpReader.NextNode(true)
+		//}
 		if !tmpReader.PeekNodeIs(true, identifierListInfixMatcher) {
 			break
 		}

parser/parser_test.go

@@ -84,42 +84,43 @@ func TestParseComments(t *testing.T) {
 			name:  "line comment with identiger",
 			input: "-- foo\nbar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 2, "\nbar")
+				testStatement(t, stmts[0], 3, "-- foo\nbar")
 
 				list := stmts[0].GetTokens()
-				testItem(t, list[0], "\n")
-				testIdentifier(t, list[1], "bar")
+				testItem(t, list[0], "-- foo")
+				testIdentifier(t, list[2], "bar")
 			},
 		},
 		{
 			name:  "range comment with identiger",
 			input: "/* foo */bar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 1, "bar")
+				testStatement(t, stmts[0], 2, "/* foo */bar")
 
 				list := stmts[0].GetTokens()
-				testIdentifier(t, list[0], "bar")
+				testIdentifier(t, list[1], "bar")
 			},
 		},
 		{
 			name:  "range comment with identiger list",
 			input: "foo, /* foo */bar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 1, "foo, bar")
+				testStatement(t, stmts[0], 1, "foo, /* foo */bar")
 
 				list := stmts[0].GetTokens()
-				testIdentifierList(t, list[0], "foo, bar")
+				testIdentifierList(t, list[0], "foo, /* foo */bar")
 			},
 		},
 		{
 			name:  "multi line range comment with identiger",
 			input: "/*\n * foo\n */\nbar",
 			checkFn: func(t *testing.T, stmts []*ast.Statement, input string) {
-				testStatement(t, stmts[0], 2, "\nbar")
+				testStatement(t, stmts[0], 3, "/*\n   foo\n */\nbar")
 
 				list := stmts[0].GetTokens()
-				testItem(t, list[0], "\n")
-				testIdentifier(t, list[1], "bar")
+				testItem(t, list[0], "/*\n   foo\n */")
+				testItem(t, list[1], "\n")
+				testIdentifier(t, list[2], "bar")
 			},
 		},
 	}

token/kind.go

@@ -20,6 +20,8 @@ const (
 	Whitespace
 	// comment node
 	Comment
+	// multiline comment node
+	MultilineComment
 	// = operator
 	Eq
 	// != or <> operator

token/lexer.go

@@ -274,7 +274,7 @@ func (t *Tokenizer) next() (Kind, interface{}, error) {
 			if err != nil {
 				return ILLEGAL, str, err
 			}
-			return Comment, str, nil
+			return MultilineComment, str, nil
 		}
 		t.Col++
 		return Div, "/", nil

token/lexer_test.go

@@ -462,7 +462,7 @@ multiline
 comment */`,
 			out: []*Token{
 				{
-					Kind:  Comment,
+					Kind:  MultilineComment,
 					Value: " test\nmultiline\ncomment ",
 					From:  Pos{Line: 0, Col: 0},
 					To:    Pos{Line: 2, Col: 10},