diff --git a/go/tools/data.go b/go/data/data.go similarity index 87% rename from go/tools/data.go rename to go/data/data.go index 390b81f..9aec39b 100644 --- a/go/tools/data.go +++ b/go/data/data.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tools +package data import ( "bytes" @@ -36,7 +36,7 @@ type ( } ) -func ReadData(url string) ([]byte, error) { +func readData(url string) ([]byte, error) { if strings.HasPrefix(url, "http") { client := http.Client{} res, err := client.Get(url) @@ -52,7 +52,11 @@ func ReadData(url string) ([]byte, error) { return os.ReadFile(url) } -func LoadQueries(data []byte) ([]Query, error) { +func LoadQueries(url string) ([]Query, error) { + data, err := readData(url) + if err != nil { + return nil, err + } seps := bytes.Split(data, []byte("\n")) queries := make([]Query, 0, len(seps)) newStmt := true @@ -94,7 +98,7 @@ func ParseQueries(qs ...Query) ([]Query, error) { realS := rs.Query s := rs.Query q := Query{} - q.Type = typ.Q_UNKNOWN + q.Type = typ.Unknown q.Line = rs.Line // a valid Query's length should be at least 3. if len(s) < 3 { @@ -102,19 +106,19 @@ func ParseQueries(qs ...Query) ([]Query, error) { } // we will skip #comment and line with zero characters here if s[0] == '#' { - q.Type = typ.Q_COMMENT + q.Type = typ.Comment } else if s[0:2] == "--" { - q.Type = typ.Q_COMMENT_WITH_COMMAND + q.Type = typ.CommentWithCommand if s[2] == ' ' { s = s[3:] } else { s = s[2:] } } else if s[0] == '\n' { - q.Type = typ.Q_EMPTY_LINE + q.Type = typ.EmptyLine } - if q.Type != typ.Q_COMMENT { + if q.Type != typ.Comment { // Calculate first word length(the command), terminated // by 'space' , '(' or 'delimiter' var i int @@ -126,7 +130,7 @@ func ParseQueries(qs ...Query) ([]Query, error) { s = s[i:] q.Query = s - if q.Type == typ.Q_UNKNOWN || q.Type == typ.Q_COMMENT_WITH_COMMAND { + if q.Type == typ.Unknown || q.Type == typ.CommentWithCommand { if err := q.getQueryType(realS); err != nil { return nil, err } @@ -143,16 +147,13 @@ func ParseQueries(qs ...Query) ([]Query, error) { func (q *Query) getQueryType(qu string) error { tp := typ.FindType(q.FirstWord) if tp > 0 { - if tp == typ.Q_ECHO || tp == typ.Q_SORTED_RESULT { - q.Query = strings.TrimSpace(q.Query) - } q.Type = tp } else { // No mysqltest command matched - if q.Type != typ.Q_COMMENT_WITH_COMMAND { + if q.Type != typ.CommentWithCommand { // A query that will sent to vitess q.Query = qu - q.Type = typ.Q_QUERY + q.Type = typ.Query } else { log.WithFields(log.Fields{"line": q.Line, "command": q.FirstWord, "arguments": q.Query}).Error("invalid command") return fmt.Errorf("invalid command %s", q.FirstWord) diff --git a/go/tester/comparing_query_runner.go b/go/tester/comparing_query_runner.go index 020f4ea..e5e0dea 100644 --- a/go/tester/comparing_query_runner.go +++ b/go/tester/comparing_query_runner.go @@ -18,10 +18,9 @@ package tester import ( "fmt" - "github.com/vitessio/vitess-tester/go/tools" - "github.com/pingcap/errors" log "github.com/sirupsen/logrus" + "github.com/vitessio/vitess-tester/go/data" "vitess.io/vitess/go/test/endtoend/utils" "vitess.io/vitess/go/vt/sqlparser" ) @@ -55,11 +54,11 @@ func newComparingQueryRunner( } } -func (nqr ComparingQueryRunner) runQuery(q tools.Query, expectedErrs bool, ast sqlparser.Statement) error { +func (nqr ComparingQueryRunner) runQuery(q data.Query, expectedErrs bool, ast sqlparser.Statement) error { return nqr.execute(q, expectedErrs, ast) } -func (nqr *ComparingQueryRunner) execute(query tools.Query, expectedErrs bool, ast sqlparser.Statement) error { +func (nqr *ComparingQueryRunner) execute(query data.Query, expectedErrs bool, ast sqlparser.Statement) error { if len(query.Query) == 0 { return nil } diff --git a/go/tester/tester.go b/go/tester/tester.go index da140dc..5324a1d 100644 --- a/go/tester/tester.go +++ b/go/tester/tester.go @@ -19,7 +19,7 @@ package tester import ( "encoding/json" "fmt" - "github.com/vitessio/vitess-tester/go/tools" + "github.com/vitessio/vitess-tester/go/data" "github.com/vitessio/vitess-tester/go/typ" "io" @@ -66,7 +66,7 @@ type ( } QueryRunner interface { - runQuery(q tools.Query, expectedErrs bool, ast sqlparser.Statement) error + runQuery(q data.Query, expectedErrs bool, ast sqlparser.Statement) error } QueryRunnerFactory interface { @@ -154,12 +154,7 @@ func (t *Tester) Run() error { if t.autoVSchema() { defer t.postProcess() } - data, err := tools.ReadData(t.name) - if err != nil { - return err - } - - queries, err := tools.LoadQueries(data) + queries, err := data.LoadQueries(t.name) if err != nil { t.reporter.AddFailure(err) return err @@ -167,24 +162,9 @@ func (t *Tester) Run() error { for _, q := range queries { switch q.Type { - // no-ops - case typ.Q_ENABLE_QUERY_LOG, - typ.Q_DISABLE_QUERY_LOG, - typ.Q_ECHO, - typ.Q_DISABLE_WARNINGS, - typ.Q_ENABLE_WARNINGS, - typ.Q_ENABLE_INFO, - typ.Q_DISABLE_INFO, - typ.Q_ENABLE_RESULT_LOG, - typ.Q_DISABLE_RESULT_LOG, - typ.Q_SORTED_RESULT, - typ.Q_REPLACE_REGEX: - // do nothing - case typ.Q_SKIP: + case typ.Skip: t.skipNext = true - case typ.Q_BEGIN_CONCURRENT, typ.Q_END_CONCURRENT, typ.Q_CONNECT, typ.Q_CONNECTION, typ.Q_DISCONNECT, typ.Q_LET, typ.Q_REPLACE_COLUMN: - t.reporter.AddFailure(fmt.Errorf("%s not supported", q.Type.String())) - case typ.Q_SKIP_IF_BELOW_VERSION: + case typ.SkipIfBelowVersion: strs := strings.Split(q.Query, " ") if len(strs) != 3 { t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.Q_SKIP_IF_BELOW_VERSION in: %v", q.Query)) @@ -197,19 +177,19 @@ func (t *Tester) Run() error { t.reporter.AddFailure(err) continue } - case typ.Q_ERROR: + case typ.Error: t.expectedErrs = true - case typ.Q_VEXPLAIN: + case typ.VExplain: strs := strings.Split(q.Query, " ") if len(strs) != 2 { - t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.Q_VEXPLAIN in: %v", q.Query)) + t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.VExplain in: %v", q.Query)) continue } t.vexplain = strs[1] - case typ.Q_WAIT_FOR_AUTHORITATIVE: + case typ.WaitForAuthoritative: t.waitAuthoritative(q.Query) - case typ.Q_QUERY: + case typ.Query: if t.vexplain != "" { result, err := t.curr.VtConn.ExecuteFetch(fmt.Sprintf("vexplain %s %s", t.vexplain, q.Query), -1, false) t.vexplain = "" @@ -221,7 +201,7 @@ func (t *Tester) Run() error { } t.runQuery(q) - case typ.Q_REMOVE_FILE: + case typ.RemoveFile: err = os.Remove(strings.TrimSpace(q.Query)) if err != nil { return errors.Annotate(err, "failed to remove file") @@ -235,7 +215,7 @@ func (t *Tester) Run() error { return nil } -func (t *Tester) runQuery(q tools.Query) { +func (t *Tester) runQuery(q data.Query) { if t.skipNext { t.skipNext = false return diff --git a/go/tester/tracer.go b/go/tester/tracer.go index f895a9e..919e072 100644 --- a/go/tester/tracer.go +++ b/go/tester/tracer.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/vitessio/vitess-tester/go/tools" + "github.com/vitessio/vitess-tester/go/data" "os" "vitess.io/vitess/go/mysql" @@ -67,7 +67,7 @@ func newTracer(traceFile *os.File, } } -func (t *Tracer) runQuery(q tools.Query, expectErr bool, ast sqlparser.Statement) error { +func (t *Tracer) runQuery(q data.Query, expectErr bool, ast sqlparser.Statement) error { if sqlparser.IsDMLStatement(ast) && t.traceFile != nil && !expectErr { // we don't want to run DMLs twice, so we just run them once while tracing var errs []error @@ -100,7 +100,7 @@ func (t *Tracer) runQuery(q tools.Query, expectErr bool, ast sqlparser.Statement } // trace writes the query and its trace (fetched from VtConn) as a JSON object into traceFile -func (t *Tracer) trace(query tools.Query) error { +func (t *Tracer) trace(query data.Query) error { // Marshal the query into JSON format for safe embedding queryJSON, err := json.Marshal(query.Query) if err != nil { diff --git a/go/typ/typ.go b/go/typ/typ.go index 997718d..c264708 100644 --- a/go/typ/typ.go +++ b/go/typ/typ.go @@ -21,211 +21,27 @@ import "strings" type CmdType int const ( - Q_CONNECTION CmdType = iota + 1 - Q_QUERY - Q_CONNECT - Q_SLEEP - Q_REAL_SLEEP - Q_INC - Q_DEC - Q_SOURCE - Q_DISCONNECT - Q_LET - Q_ECHO - Q_WHILE - Q_END_BLOCK - Q_SYSTEM - Q_RESULT - Q_REQUIRE - Q_SAVE_MASTER_POS - Q_SYNC_WITH_MASTER - Q_SYNC_SLAVE_WITH_MASTER - Q_ERROR - Q_SEND - Q_REAP - Q_DIRTY_CLOSE - Q_REPLACE - Q_REPLACE_COLUMN - Q_PING - Q_EVAL - Q_EVAL_RESULT - Q_ENABLE_QUERY_LOG - Q_DISABLE_QUERY_LOG - Q_ENABLE_RESULT_LOG - Q_DISABLE_RESULT_LOG - Q_ENABLE_CONNECT_LOG - Q_DISABLE_CONNECT_LOG - Q_WAIT_FOR_SLAVE_TO_STOP - Q_ENABLE_WARNINGS - Q_DISABLE_WARNINGS - Q_ENABLE_INFO - Q_DISABLE_INFO - Q_ENABLE_SESSION_TRACK_INFO - Q_DISABLE_SESSION_TRACK_INFO - Q_ENABLE_METADATA - Q_DISABLE_METADATA - Q_EXEC - Q_EXECW - Q_DELIMITER - Q_DISABLE_ABORT_ON_ERROR - Q_ENABLE_ABORT_ON_ERROR - Q_DISPLAY_VERTICAL_RESULTS - Q_DISPLAY_HORIZONTAL_RESULTS - Q_QUERY_VERTICAL - Q_QUERY_HORIZONTAL - Q_SORTED_RESULT - Q_LOWERCASE - Q_START_TIMER - Q_END_TIMER - Q_CHARACTER_SET - Q_DISABLE_PS_PROTOCOL - Q_ENABLE_PS_PROTOCOL - Q_DISABLE_RECONNECT - Q_ENABLE_RECONNECT - Q_IF - Q_DISABLE_PARSING - Q_ENABLE_PARSING - Q_REPLACE_REGEX - Q_REPLACE_NUMERIC_ROUND - Q_REMOVE_FILE - Q_FILE_EXIST - Q_WRITE_FILE - Q_COPY_FILE - Q_PERL - Q_DIE - Q_EXIT - Q_SKIP - Q_CHMOD_FILE - Q_APPEND_FILE - Q_CAT_FILE - Q_DIFF_FILES - Q_SEND_QUIT - Q_CHANGE_USER - Q_MKDIR - Q_RMDIR - Q_LIST_FILES - Q_LIST_FILES_WRITE_FILE - Q_LIST_FILES_APPEND_FILE - Q_SEND_SHUTDOWN - Q_SHUTDOWN_SERVER - Q_RESULT_FORMAT_VERSION - Q_MOVE_FILE - Q_REMOVE_FILES_WILDCARD - Q_SEND_EVAL - Q_OUTPUT /* redirect output to a file */ - Q_RESET_CONNECTION - Q_SINGLE_QUERY - Q_BEGIN_CONCURRENT - Q_END_CONCURRENT - Q_UNKNOWN - Q_COMMENT - Q_COMMENT_WITH_COMMAND - Q_EMPTY_LINE - Q_SKIP_IF_BELOW_VERSION - Q_VEXPLAIN - Q_WAIT_FOR_AUTHORITATIVE + Query CmdType = iota + Error + RemoveFile + Skip + Unknown + Comment + CommentWithCommand + EmptyLine + SkipIfBelowVersion + VExplain + WaitForAuthoritative ) var commandMap = map[string]CmdType{ - "connection": Q_CONNECTION, - "query": Q_QUERY, - "connect": Q_CONNECT, - "sleep": Q_SLEEP, - "real_sleep": Q_REAL_SLEEP, - "inc": Q_INC, - "dec": Q_DEC, - "source": Q_SOURCE, - "disconnect": Q_DISCONNECT, - "let": Q_LET, - "echo": Q_ECHO, - "while": Q_WHILE, - "end": Q_END_BLOCK, - "system": Q_SYSTEM, - "result": Q_RESULT, - "require": Q_REQUIRE, - "save_master_pos": Q_SAVE_MASTER_POS, - "sync_with_master": Q_SYNC_WITH_MASTER, - "sync_slave_with_master": Q_SYNC_SLAVE_WITH_MASTER, - "error": Q_ERROR, - "send": Q_SEND, - "reap": Q_REAP, - "dirty_close": Q_DIRTY_CLOSE, - "replace_result": Q_REPLACE, - "replace_column": Q_REPLACE_COLUMN, - "ping": Q_PING, - "eval": Q_EVAL, - "eval_result": Q_EVAL_RESULT, - "enable_query_log": Q_ENABLE_QUERY_LOG, - "disable_query_log": Q_DISABLE_QUERY_LOG, - "enable_result_log": Q_ENABLE_RESULT_LOG, - "disable_result_log": Q_DISABLE_RESULT_LOG, - "enable_connect_log": Q_ENABLE_CONNECT_LOG, - "disable_connect_log": Q_DISABLE_CONNECT_LOG, - "wait_for_slave_to_stop": Q_WAIT_FOR_SLAVE_TO_STOP, - "enable_warnings": Q_ENABLE_WARNINGS, - "disable_warnings": Q_DISABLE_WARNINGS, - "enable_info": Q_ENABLE_INFO, - "disable_info": Q_DISABLE_INFO, - "enable_session_track_info": Q_ENABLE_SESSION_TRACK_INFO, - "disable_session_track_info": Q_DISABLE_SESSION_TRACK_INFO, - "enable_metadata": Q_ENABLE_METADATA, - "disable_metadata": Q_DISABLE_METADATA, - "exec": Q_EXEC, - "execw": Q_EXECW, - "delimiter": Q_DELIMITER, - "disable_abort_on_error": Q_DISABLE_ABORT_ON_ERROR, - "enable_abort_on_error": Q_ENABLE_ABORT_ON_ERROR, - "vertical_results": Q_DISPLAY_VERTICAL_RESULTS, - "horizontal_results": Q_DISPLAY_HORIZONTAL_RESULTS, - "query_vertical": Q_QUERY_VERTICAL, - "query_horizontal": Q_QUERY_HORIZONTAL, - "sorted_result": Q_SORTED_RESULT, - "lowercase_result": Q_LOWERCASE, - "start_timer": Q_START_TIMER, - "end_timer": Q_END_TIMER, - "character_set": Q_CHARACTER_SET, - "disable_ps_protocol": Q_DISABLE_PS_PROTOCOL, - "enable_ps_protocol": Q_ENABLE_PS_PROTOCOL, - "disable_reconnect": Q_DISABLE_RECONNECT, - "enable_reconnect": Q_ENABLE_RECONNECT, - "if": Q_IF, - "disable_parsing": Q_DISABLE_PARSING, - "enable_parsing": Q_ENABLE_PARSING, - "replace_regex": Q_REPLACE_REGEX, - "replace_numeric_round": Q_REPLACE_NUMERIC_ROUND, - "remove_file": Q_REMOVE_FILE, - "file_exists": Q_FILE_EXIST, - "write_file": Q_WRITE_FILE, - "copy_file": Q_COPY_FILE, - "perl": Q_PERL, - "die": Q_DIE, - "exit": Q_EXIT, - "skip": Q_SKIP, - "chmod": Q_CHMOD_FILE, - "append_file": Q_APPEND_FILE, - "cat_file": Q_CAT_FILE, - "diff_files": Q_DIFF_FILES, - "send_quit": Q_SEND_QUIT, - "change_user": Q_CHANGE_USER, - "mkdir": Q_MKDIR, - "rmdir": Q_RMDIR, - "list_files": Q_LIST_FILES, - "list_files_write_file": Q_LIST_FILES_WRITE_FILE, - "list_files_append_file": Q_LIST_FILES_APPEND_FILE, - "send_shutdown": Q_SEND_SHUTDOWN, - "shutdown_server": Q_SHUTDOWN_SERVER, - "result_format": Q_RESULT_FORMAT_VERSION, - "move_file": Q_MOVE_FILE, - "remove_files_wildcard": Q_REMOVE_FILES_WILDCARD, - "send_eval": Q_SEND_EVAL, - "output": Q_OUTPUT, - "reset_connection": Q_RESET_CONNECTION, - "single_query": Q_SINGLE_QUERY, - "begin_concurrent": Q_BEGIN_CONCURRENT, - "end_concurrent": Q_END_CONCURRENT, - "skip_if_below_version": Q_SKIP_IF_BELOW_VERSION, - "vexplain": Q_VEXPLAIN, - "wait_authoritative": Q_WAIT_FOR_AUTHORITATIVE, + "query": Query, + "error": Error, + "remove_file": RemoveFile, + "skip": Skip, + "skip_if_below_version": SkipIfBelowVersion, + "vexplain": VExplain, + "wait_authoritative": WaitForAuthoritative, } func (cmd CmdType) String() string {