From 42e937fdbb5812b0dc79d55749359424b9e82f68 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Tue, 27 Sep 2022 19:18:12 +1000 Subject: [PATCH] Add the seed for specification tests. The goal is to have a single lexer definition that exercises all the functionality of the stateful lexer and generated equivalent. See #264 --- cmd/participle/files/codegen.go.tmpl | 1 + cmd/participle/gen_lexer_cmd.go | 12 +- lexer/internal/spec/spec_codegen_test.go | 14 +++ lexer/internal/spec/spec_test.go | 153 +++++++++++++++++++++++ scripts/participle | 2 +- 5 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 lexer/internal/spec/spec_codegen_test.go create mode 100644 lexer/internal/spec/spec_test.go diff --git a/cmd/participle/files/codegen.go.tmpl b/cmd/participle/files/codegen.go.tmpl index 9a601da0..9a2da546 100644 --- a/cmd/participle/files/codegen.go.tmpl +++ b/cmd/participle/files/codegen.go.tmpl @@ -12,6 +12,7 @@ import ( ) var _ syntax.Op +const _ = utf8.RuneError var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{} diff --git a/cmd/participle/gen_lexer_cmd.go b/cmd/participle/gen_lexer_cmd.go index d84e51fc..730319da 100644 --- a/cmd/participle/gen_lexer_cmd.go +++ b/cmd/participle/gen_lexer_cmd.go @@ -19,7 +19,7 @@ type genLexerCmd struct { Name string `help:"Name of the lexer."` Output string `short:"o" help:"Output file."` Package string `arg:"" required:"" help:"Go package for generated code."` - Lexer string `arg:"" required:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer."` + Lexer string `arg:"" required:"" default:"-" type:"existingfile" help:"JSON representation of a Participle lexer (read from stdin if omitted)."` } func (c *genLexerCmd) Help() string { @@ -52,7 +52,15 @@ func (c *genLexerCmd) Run() error { if err != nil { return err } - err = generateLexer(os.Stdout, c.Package, def, c.Name) + out := os.Stdout + if c.Output != "" { + out, err = os.Create(c.Output) + if err != nil { + return err + } + defer out.Close() + } + err = generateLexer(out, c.Package, def, c.Name) if err != nil { return err } diff --git a/lexer/internal/spec/spec_codegen_test.go b/lexer/internal/spec/spec_codegen_test.go new file mode 100644 index 00000000..a6fd7cf9 --- /dev/null +++ b/lexer/internal/spec/spec_codegen_test.go @@ -0,0 +1,14 @@ +//go:build generated + +package spec_test + +import ( + "testing" + + "github.com/alecthomas/participle/v2/lexer/internal/spec" +) + +// This should only be run by TestLexerSpecGenerated. +func TestLexerSpecGeneratedInternal(t *testing.T) { + testLexer(t, spec.GeneratedSpecLexer) +} diff --git a/lexer/internal/spec/spec_test.go b/lexer/internal/spec/spec_test.go new file mode 100644 index 00000000..b3b5e037 --- /dev/null +++ b/lexer/internal/spec/spec_test.go @@ -0,0 +1,153 @@ +package spec_test + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/participle/v2/lexer" +) + +var specLexer = lexer.MustStateful(lexer.Rules{ + "Root": { + {"String", `"`, lexer.Push("String")}, + {"Heredoc", `<<(\w+\b)`, lexer.Push("Heredoc")}, + }, + "String": { + {"Escaped", `\\.`, nil}, + {"StringEnd", `"`, lexer.Pop()}, + {"Expr", `\${`, lexer.Push("Expr")}, + {"Char", `[^$"\\]+`, nil}, + }, + "Expr": { + lexer.Include("Root"), + {`Whitespace`, `\s+`, nil}, + {`Oper`, `[-+/*%]`, nil}, + {"Ident", `\w+`, lexer.Push("Reference")}, + {"ExprEnd", `}`, lexer.Pop()}, + }, + "Reference": { + {"Dot", `\.`, nil}, + {"Ident", `\w+`, nil}, + lexer.Return(), + }, + "Heredoc": { + {"End", `\b\1\b`, lexer.Pop()}, + lexer.Include("Expr"), + }, +}) + +type token struct { + Type string + Value string +} + +func testLexer(t *testing.T, lex lexer.Definition) { + t.Helper() + tests := []struct { + name string + input string + expected []token + }{ + {"NestedString", `"${"Hello ${name + "!"}"}"`, []token{ + {"String", "\""}, + {"Expr", "${"}, + {"String", "\""}, + {"Char", "Hello "}, + {"Expr", "${"}, + {"Ident", "name"}, + {"Whitespace", " "}, + {"Oper", "+"}, + {"Whitespace", " "}, + {"String", "\""}, + {"Char", "!"}, + {"StringEnd", "\""}, + {"ExprEnd", "}"}, + {"StringEnd", "\""}, + {"ExprEnd", "}"}, + {"StringEnd", "\""}, + }}, + {"Reference", `"${user.name}"`, []token{ + {"String", "\""}, + {"Expr", "${"}, + {"Ident", "user"}, + {"Dot", "."}, + {"Ident", "name"}, + {"ExprEnd", "}"}, + {"StringEnd", "\""}, + }}, + } + symbols := lexer.SymbolsByRune(lex) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l, err := lex.Lex(test.name, strings.NewReader(test.input)) + assert.NoError(t, err) + tokens, err := lexer.ConsumeAll(l) + assert.NoError(t, err) + actual := make([]token, len(tokens)-1) + for i, t := range tokens { + if t.Type == lexer.EOF { + continue + } + actual[i] = token{Type: symbols[t.Type], Value: t.Value} + } + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestLexerSpecGenerated(t *testing.T) { + genLexer(t) + args := []string{"test", "-run", "TestLexerSpecGeneratedInternal", "-tags", "generated"} + // Propagate test flags. + flag.CommandLine.VisitAll(func(f *flag.Flag) { + if f.Value.String() != f.DefValue { + args = append(args, fmt.Sprintf("-%s=%s", f.Name, f.Value.String())) + } + }) + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + assert.NoError(t, err) +} + +func TestLexerSpec(t *testing.T) { + testLexer(t, specLexer) +} + +func genLexer(t *testing.T) { + t.Helper() + lexerJSON, err := json.Marshal(specLexer) + assert.NoError(t, err) + cwd, err := os.Getwd() + assert.NoError(t, err) + generatedSpecLexer := filepath.Join(cwd, "spec_lexer_gen.go") + t.Cleanup(func() { + _ = os.Remove(generatedSpecLexer) + }) + cmd := exec.Command( + "../../../scripts/participle", + "gen", "lexer", "spec", + "--name", "GeneratedSpec", + "--output", generatedSpecLexer) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + w, err := cmd.StdinPipe() + assert.NoError(t, err) + defer w.Close() + err = cmd.Start() + assert.NoError(t, err) + _, err = w.Write(lexerJSON) + assert.NoError(t, err) + err = w.Close() + assert.NoError(t, err) + err = cmd.Wait() + assert.NoError(t, err) +} diff --git a/scripts/participle b/scripts/participle index eccdee5d..d1744c9a 100755 --- a/scripts/participle +++ b/scripts/participle @@ -1,4 +1,4 @@ #!/bin/bash set -euo pipefail -(cd "$(dirname $0)/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle) +(cd "$(dirname "$0")/../cmd/participle" && go install github.com/alecthomas/participle/v2/cmd/participle) exec "$(go env GOBIN)/participle" "$@"