Merge pull request #26 from vitessio/query-runner

introduce QueryRunner interface and start using it

main.go

@@ -16,9 +16,13 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
+	"time"
 
 	log "github.com/sirupsen/logrus"
+	"vitess.io/vitess/go/test/endtoend/cluster"
 
 	"github.com/vitessio/vitess-tester/src/cmd"
 	vitess_tester "github.com/vitessio/vitess-tester/src/vitess-tester"
@@ -92,13 +96,50 @@ func main() {
 	if xunit {
 		reporterSuite = vitess_tester.NewXMLTestSuite()
 	} else {
-		reporterSuite = vitess_tester.NewFileReporterSuite()
+		reporterSuite = vitess_tester.NewFileReporterSuite(getVschema(clusterInstance))
 	}
-	failed := cmd.ExecuteTests(clusterInstance, vtParams, mysqlParams, tests, reporterSuite, ksNames, vschemaFile, vtexplainVschemaFile, olap, traceFile)
+	failed := cmd.ExecuteTests(clusterInstance, vtParams, mysqlParams, tests, reporterSuite, ksNames, vschemaFile, vtexplainVschemaFile, olap, getQueryRunnerFactory())
 	outputFile := reporterSuite.Close()
 	if failed {
 		log.Errorf("some tests failed 😭\nsee errors in %v", outputFile)
 		os.Exit(1)
 	}
 	println("Great, All tests passed")
 }
+
+func getQueryRunnerFactory() vitess_tester.QueryRunnerFactory {
+	inner := vitess_tester.ComparingQueryRunnerFactory{}
+	if traceFile == "" {
+		return inner
+	}
+
+	var err error
+	writer, err := os.Create(traceFile)
+	if err != nil {
+		panic(err)
+	}
+	_, err = writer.Write([]byte("["))
+	if err != nil {
+		panic(err.Error())
+	}
+	return vitess_tester.NewTracerFactory(writer, inner)
+}
+
+func getVschema(clusterInstance *cluster.LocalProcessCluster) func() []byte {
+	return func() []byte {
+		httpClient := &http.Client{Timeout: 5 * time.Second}
+		resp, err := httpClient.Get(clusterInstance.VtgateProcess.VSchemaURL)
+		if err != nil {
+			log.Errorf(err.Error())
+			return nil
+		}
+		defer resp.Body.Close()
+		res, err := io.ReadAll(resp.Body)
+		if err != nil {
+			log.Errorf(err.Error())
+			return nil
+		}
+
+		return res
+	}
+}

src/cmd/cmd.go

@@ -53,28 +53,16 @@ func ExecuteTests(
 	ksNames []string,
 	vschemaFile, vtexplainVschemaFile string,
 	olap bool,
-	traceFile string,
+	factory vitess_tester.QueryRunnerFactory,
 ) (failed bool) {
 	vschemaF := vschemaFile
 	if vschemaF == "" {
 		vschemaF = vtexplainVschemaFile
 	}
-	var writer *os.File
-	if traceFile != "" {
-		// create the file and store the writer in the Tester struct
-		var err error
-		writer, err = os.Create(traceFile)
-		if err != nil {
-			panic(err)
-		}
-		_, err = writer.Write([]byte("["))
-		if err != nil {
-			panic(err.Error())
-		}
-	}
+
 	for _, name := range fileNames {
 		errReporter := s.NewReporterForFile(name)
-		vTester := vitess_tester.NewTester(name, errReporter, clusterInstance, vtParams, mysqlParams, olap, ksNames, vschema, vschemaF, writer)
+		vTester := vitess_tester.NewTester(name, errReporter, clusterInstance, vtParams, mysqlParams, olap, ksNames, vschema, vschemaF, factory)
 		err := vTester.Run()
 		if err != nil {
 			failed = true
@@ -83,17 +71,10 @@ func ExecuteTests(
 		failed = failed || errReporter.Failed()
 		s.CloseReportForFile()
 	}
-	if writer != nil {
-		_, err := writer.Write([]byte("]"))
-		if err != nil {
-			panic(err.Error())
-		}
-		err = writer.Close()
-		if err != nil {
-			panic(err.Error())
-		}
-	}
-	return
+
+	factory.Close()
+
+	return failed
 }
 
 func SetupCluster(

src/vitess-tester/comparing_query_runner.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vitess_tester
+
+import (
+	"fmt"
+
+	"github.com/pingcap/errors"
+	log "github.com/sirupsen/logrus"
+	"vitess.io/vitess/go/test/endtoend/utils"
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+type (
+	// ComparingQueryRunner is a QueryRunner that compares the results of the queries between MySQL and Vitess
+	ComparingQueryRunner struct {
+		reporter          Reporter
+		handleCreateTable CreateTableHandler
+		comparer          utils.MySQLCompare
+	}
+	CreateTableHandler          func(create *sqlparser.CreateTable) func()
+	ComparingQueryRunnerFactory struct{}
+)
+
+func (f ComparingQueryRunnerFactory) Close() {}
+
+func (f ComparingQueryRunnerFactory) NewQueryRunner(reporter Reporter, handleCreateTable CreateTableHandler, comparer utils.MySQLCompare) QueryRunner {
+	return newComparingQueryRunner(reporter, handleCreateTable, comparer)
+}
+
+func newComparingQueryRunner(
+	reporter Reporter,
+	handleCreateTable CreateTableHandler,
+	comparer utils.MySQLCompare,
+) *ComparingQueryRunner {
+	return &ComparingQueryRunner{
+		reporter:          reporter,
+		handleCreateTable: handleCreateTable,
+		comparer:          comparer,
+	}
+}
+
+func (nqr ComparingQueryRunner) runQuery(q query, expectedErrs bool, ast sqlparser.Statement) error {
+	return nqr.execute(q, expectedErrs, ast)
+}
+
+func (nqr *ComparingQueryRunner) execute(query query, expectedErrs bool, ast sqlparser.Statement) error {
+	if len(query.Query) == 0 {
+		return nil
+	}
+
+	if err := nqr.executeStmt(query.Query, ast, expectedErrs); err != nil {
+		return errors.Trace(errors.Errorf("run \"%v\" at line %d err %v", query.Query, query.Line, err))
+	}
+	// clear expected errors after we execute
+	expectedErrs = false
+
+	return nil
+}
+
+func (nqr *ComparingQueryRunner) executeStmt(query string, ast sqlparser.Statement, expectedErrs bool) (err error) {
+	_, commentOnly := ast.(*sqlparser.CommentOnly)
+	if commentOnly {
+		return nil
+	}
+
+	log.Debugf("executeStmt: %s", query)
+	create, isCreateStatement := ast.(*sqlparser.CreateTable)
+	if isCreateStatement && !expectedErrs {
+		closer := nqr.handleCreateTable(create)
+		defer func() {
+			if err == nil {
+				closer()
+			}
+		}()
+	}
+
+	switch {
+	case expectedErrs:
+		_, err := nqr.comparer.ExecAllowAndCompareError(query, utils.CompareOptions{CompareColumnNames: true})
+		if err == nil {
+			// If we expected an error, but didn't get one, return an error
+			return fmt.Errorf("expected error, but got none")
+		}
+	default:
+		_ = nqr.comparer.Exec(query)
+	}
+	return nil
+}

src/vitess-tester/reporter.go

@@ -23,27 +23,32 @@ import (
 	"path"
 	"strings"
 	"time"
+
+	"vitess.io/vitess/go/test/endtoend/utils"
 )
 
 type Suite interface {
 	NewReporterForFile(name string) Reporter
 	CloseReportForFile()
-	Close() string
+	Close() string // returns the path to the file or directory with files
 }
 
 type Reporter interface {
+	utils.TestingT
 	AddTestCase(query string, lineNo int)
 	EndTestCase()
-	AddFailure(vschema []byte, err error)
+	AddFailure(err error)
 	AddInfo(info string)
 	Report() string
 	Failed() bool
 }
 
-type FileReporterSuite struct{}
+type FileReporterSuite struct {
+	getVschema func() []byte
+}
 
 func (frs *FileReporterSuite) NewReporterForFile(name string) Reporter {
-	return newFileReporter(name)
+	return newFileReporter(name, frs.getVschema)
 }
 
 func (frs *FileReporterSuite) CloseReportForFile() {}
@@ -52,8 +57,10 @@ func (frs *FileReporterSuite) Close() string {
 	return "errors"
 }
 
-func NewFileReporterSuite() *FileReporterSuite {
-	return &FileReporterSuite{}
+func NewFileReporterSuite(getVschema func() []byte) *FileReporterSuite {
+	return &FileReporterSuite{
+		getVschema: getVschema,
+	}
 }
 
 type FileReporter struct {
@@ -69,12 +76,15 @@ type FileReporter struct {
 	failureCount int
 	queryCount   int
 	successCount int
+
+	getVschema func() []byte
 }
 
-func newFileReporter(name string) *FileReporter {
+func newFileReporter(name string, getVschema func() []byte) *FileReporter {
 	return &FileReporter{
-		name:      name,
-		startTime: time.Now(),
+		name:       name,
+		startTime:  time.Now(),
+		getVschema: getVschema,
 	}
 
 }
@@ -115,7 +125,7 @@ func (e *FileReporter) EndTestCase() {
 	}
 }
 
-func (e *FileReporter) AddFailure(vschema []byte, err error) {
+func (e *FileReporter) AddFailure(err error) {
 	e.failureCount++
 	e.currentQueryFailed = true
 	if e.currentQuery == "" {
@@ -130,7 +140,7 @@ func (e *FileReporter) AddFailure(vschema []byte, err error) {
 		panic("failed to write error file\n" + err.Error())
 	}
 
-	e.createVSchemaDump(vschema)
+	e.createVSchemaDump()
 }
 
 func (e *FileReporter) AddInfo(info string) {
@@ -162,14 +172,14 @@ func (e *FileReporter) createErrorFileFor() *os.File {
 	return file
 }
 
-func (e *FileReporter) createVSchemaDump(vschema []byte) {
+func (e *FileReporter) createVSchemaDump() {
 	errorDir := e.errorDir()
 	err := os.MkdirAll(errorDir, PERM)
 	if err != nil {
 		panic("failed to create vschema directory\n" + err.Error())
 	}
 
-	err = os.WriteFile(path.Join(errorDir, "vschema.json"), vschema, PERM)
+	err = os.WriteFile(path.Join(errorDir, "vschema.json"), e.getVschema(), PERM)
 	if err != nil {
 		panic("failed to write vschema\n" + err.Error())
 	}
@@ -189,4 +199,14 @@ func (e *FileReporter) errorDir() string {
 	return path.Join("errors", errFileName)
 }
 
+func (e *FileReporter) Errorf(format string, args ...interface{}) {
+	e.AddFailure(fmt.Errorf(format, args...))
+}
+
+func (e *FileReporter) FailNow() {
+	// we don't need to do anything here
+}
+
+func (e *FileReporter) Helper() {}
+
 var _ Reporter = (*FileReporter)(nil)