Add a syntax to support embed custom parser code during parsing (a proposal with demo code)

ast/ast_code.go

@@ -0,0 +1,42 @@
+package ast
+
+import "fmt"
+
+// CustomParserCodeExpr supports custom parser.
+type CustomParserCodeExpr struct {
+	p      Pos
+	Code   *CodeBlock
+	FuncIx int
+	X      int
+}
+
+var _ Expression = (*CustomParserCodeExpr)(nil)
+
+// NewCustomParserCodeExpr creates a new state (#) code expression at the specified
+// position.
+func NewCustomParserCodeExpr(p Pos) *CustomParserCodeExpr {
+	return &CustomParserCodeExpr{p: p, X: 1}
+}
+
+// Pos returns the starting position of the node.
+func (s *CustomParserCodeExpr) Pos() Pos { return s.p }
+
+// String returns the textual representation of a node.
+func (s *CustomParserCodeExpr) String() string {
+	return fmt.Sprintf("%s: %T{Code: %v}", s.p, s, s.Code)
+}
+
+// NullableVisit recursively determines whether an object is nullable.
+func (s *CustomParserCodeExpr) NullableVisit(rules map[string]*Rule) bool {
+	return true
+}
+
+// IsNullable returns the nullable attribute of the node.
+func (s *CustomParserCodeExpr) IsNullable() bool {
+	return true
+}
+
+// InitialNames returns names of nodes with which an expression can begin.
+func (s *CustomParserCodeExpr) InitialNames() map[string]struct{} {
+	return make(map[string]struct{})
+}

ast/ast_walk.go

@@ -59,6 +59,8 @@ func Walk(v Visitor, expr Expression) {
 		}
 	case *StateCodeExpr:
 		// Nothing to do
+	case *CustomParserCodeExpr:
+		// Nothing to do
 	case *ZeroOrMoreExpr:
 		Walk(v, expr.Expr)
 	case *ZeroOrOneExpr:

builder/builder.go

@@ -30,6 +30,10 @@ var (
 	onStateFuncTemplate = `func (%s *current) %s(%s) (error) {
 %s
 }
+`
+	onStateFuncTemplate2 = `func (%s *current) %s(p *parser, %s) (any, bool, error) {
+%s
+}
 `
 	callFuncTemplate = `func (p *parser) call%s() (any, error) {
 	stack := p.vstack[len(p.vstack)-1]
@@ -48,6 +52,12 @@ var (
     _ = stack
     return p.cur.%[1]s(%s)
 }
+`
+	callStateFuncTemplate2 = `func (p *parser) call%s() (any, bool, error) {
+    stack := p.vstack[len(p.vstack)-1]
+    _ = stack
+    return p.cur.%[1]s(p, %s)
+}
 `
 )
 
@@ -243,6 +253,8 @@ func (b *builder) writeExpr(expr ast.Expression) {
 		b.writeSeqExpr(expr)
 	case *ast.StateCodeExpr:
 		b.writeStateCodeExpr(expr)
+	case *ast.CustomParserCodeExpr:
+		b.writeCustomParserCodeExpr(expr)
 	case *ast.ThrowExpr:
 		b.writeThrowExpr(expr)
 	case *ast.ZeroOrMoreExpr:
@@ -563,6 +575,22 @@ func (b *builder) writeStateCodeExpr(state *ast.StateCodeExpr) {
 	b.writelnf("},")
 }
 
+func (b *builder) writeCustomParserCodeExpr(state *ast.CustomParserCodeExpr) {
+	if state == nil {
+		b.writelnf("nil,")
+		return
+	}
+	b.globalState = true
+	b.writelnf("&customParserCodeExpr{")
+	pos := state.Pos()
+	if state.FuncIx == 0 {
+		state.FuncIx = b.exprIndex
+	}
+	b.writelnf("\tpos: position{line: %d, col: %d, offset: %d},", pos.Line, pos.Col, pos.Off)
+	b.writelnf("\trun: (*parser).call%s,", b.funcName(state.FuncIx))
+	b.writelnf("},")
+}
+
 func (b *builder) writeThrowExpr(throw *ast.ThrowExpr) {
 	if throw == nil {
 		b.writelnf("nil,")
@@ -684,6 +712,9 @@ func (b *builder) writeExprCode(expr ast.Expression) {
 	case *ast.StateCodeExpr:
 		b.writeStateCodeExprCode(expr)
 
+	case *ast.CustomParserCodeExpr:
+		b.writeCustomParserCodeExprCode(expr)
+
 	case *ast.ZeroOrMoreExpr:
 		b.pushArgsSet()
 		b.writeExprCode(expr.Expr)
@@ -736,6 +767,16 @@ func (b *builder) writeStateCodeExprCode(state *ast.StateCodeExpr) {
 	}
 }
 
+func (b *builder) writeCustomParserCodeExprCode(state *ast.CustomParserCodeExpr) {
+	if state == nil {
+		return
+	}
+	if state.FuncIx > 0 {
+		b.writeFunc(state.FuncIx, state.Code, callStateFuncTemplate2, onStateFuncTemplate2)
+		state.FuncIx = 0 // already rendered, prevent duplicates
+	}
+}
+
 func (b *builder) writeFunc(funcIx int, code *ast.CodeBlock, callTpl, funcTpl string) {
 	if code == nil {
 		return

builder/static_code.go

@@ -366,6 +366,12 @@ type litMatcher struct {
 	want       string
 }
 
+// {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }}
+type customParserCodeExpr struct {
+	pos position
+	run func(*parser) (any, bool, error)
+}
+
 // {{ if .Nolint }} nolint: structcheck {{else}} ==template== {{ end }}
 type charClassMatcher struct {
 	pos             position
@@ -1103,6 +1109,8 @@ func (p *parser) parseExpr(expr any) (any, bool) {
 		val, ok = p.parseCharClassMatcher(expr)
 	case *choiceExpr:
 		val, ok = p.parseChoiceExpr(expr)
+	case *customParserCodeExpr:
+		val, ok = p.parseCustomParserCodeExpr(expr)
 	case *labeledExpr:
 		val, ok = p.parseLabeledExpr(expr)
 	case *litMatcher:
@@ -1381,6 +1389,21 @@ func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) {
 	return val, ok
 }
 
+func (p *parser) parseCustomParserCodeExpr(code *customParserCodeExpr) (any, bool) {
+	// ==template== {{ if not .Optimize }}
+	if p.debug {
+		defer p.out(p.in("parseCustomParserCodeExpr"))
+	}
+
+	// {{ end }} ==template==
+	val, ok, err := code.run(p)
+	if err != nil {
+		p.addErr(err)
+		return nil, true
+	}
+	return val, ok
+}
+
 func (p *parser) parseLitMatcher(lit *litMatcher) (any, bool) {
 	// ==template== {{ if not .Optimize }}
 	if p.debug {

grammar/pigeon.peg

@@ -176,6 +176,11 @@ SemanticPredExpr ← op:SemanticPredOp __ code:CodeBlock {
         and.Code = code.(*ast.CodeBlock)
         return and, nil
 
+    case "*":
+        state := ast.NewCustomParserCodeExpr(c.astPos())
+        state.Code = code.(*ast.CodeBlock)
+        return state, nil
+
     // case "!":
     default:
         not := ast.NewNotCodeExpr(c.astPos())
@@ -184,7 +189,7 @@ SemanticPredExpr ← op:SemanticPredOp __ code:CodeBlock {
 
     }
 }
-SemanticPredOp ← ( '#' / '&' / '!' ) {
+SemanticPredOp ← ( '#' / '&' / '!' / '*' ) {
     return string(c.text), nil
 }