Skip to content

Commit

Permalink
more cleanups
Browse files Browse the repository at this point in the history
Signed-off-by: Andres Taylor <[email protected]>
systay committed Oct 2, 2024

Verified

This commit was signed with the committer’s verified signature.
systay Andrés Taylor
1 parent 4b9a20c commit 33030b3
Showing 5 changed files with 51 additions and 255 deletions.
29 changes: 15 additions & 14 deletions go/tools/data.go → go/data/data.go
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package tools
package data

import (
"bytes"
@@ -36,7 +36,7 @@ type (
}
)

func ReadData(url string) ([]byte, error) {
func readData(url string) ([]byte, error) {
if strings.HasPrefix(url, "http") {
client := http.Client{}
res, err := client.Get(url)
@@ -52,7 +52,11 @@ func ReadData(url string) ([]byte, error) {
return os.ReadFile(url)
}

func LoadQueries(data []byte) ([]Query, error) {
func LoadQueries(url string) ([]Query, error) {
data, err := readData(url)
if err != nil {
return nil, err
}
seps := bytes.Split(data, []byte("\n"))
queries := make([]Query, 0, len(seps))
newStmt := true
@@ -94,27 +98,27 @@ func ParseQueries(qs ...Query) ([]Query, error) {
realS := rs.Query
s := rs.Query
q := Query{}
q.Type = typ.Q_UNKNOWN
q.Type = typ.Unknown
q.Line = rs.Line
// a valid Query's length should be at least 3.
if len(s) < 3 {
continue
}
// we will skip #comment and line with zero characters here
if s[0] == '#' {
q.Type = typ.Q_COMMENT
q.Type = typ.Comment
} else if s[0:2] == "--" {
q.Type = typ.Q_COMMENT_WITH_COMMAND
q.Type = typ.CommentWithCommand
if s[2] == ' ' {
s = s[3:]
} else {
s = s[2:]
}
} else if s[0] == '\n' {
q.Type = typ.Q_EMPTY_LINE
q.Type = typ.EmptyLine
}

if q.Type != typ.Q_COMMENT {
if q.Type != typ.Comment {
// Calculate first word length(the command), terminated
// by 'space' , '(' or 'delimiter'
var i int
@@ -126,7 +130,7 @@ func ParseQueries(qs ...Query) ([]Query, error) {
s = s[i:]

q.Query = s
if q.Type == typ.Q_UNKNOWN || q.Type == typ.Q_COMMENT_WITH_COMMAND {
if q.Type == typ.Unknown || q.Type == typ.CommentWithCommand {
if err := q.getQueryType(realS); err != nil {
return nil, err
}
@@ -143,16 +147,13 @@ func ParseQueries(qs ...Query) ([]Query, error) {
func (q *Query) getQueryType(qu string) error {
tp := typ.FindType(q.FirstWord)
if tp > 0 {
if tp == typ.Q_ECHO || tp == typ.Q_SORTED_RESULT {
q.Query = strings.TrimSpace(q.Query)
}
q.Type = tp
} else {
// No mysqltest command matched
if q.Type != typ.Q_COMMENT_WITH_COMMAND {
if q.Type != typ.CommentWithCommand {
// A query that will sent to vitess
q.Query = qu
q.Type = typ.Q_QUERY
q.Type = typ.Query
} else {
log.WithFields(log.Fields{"line": q.Line, "command": q.FirstWord, "arguments": q.Query}).Error("invalid command")
return fmt.Errorf("invalid command %s", q.FirstWord)
7 changes: 3 additions & 4 deletions go/tester/comparing_query_runner.go
Original file line number Diff line number Diff line change
@@ -18,10 +18,9 @@ package tester

import (
"fmt"
"github.com/vitessio/vitess-tester/go/tools"

"github.com/pingcap/errors"
log "github.com/sirupsen/logrus"
"github.com/vitessio/vitess-tester/go/data"
"vitess.io/vitess/go/test/endtoend/utils"
"vitess.io/vitess/go/vt/sqlparser"
)
@@ -55,11 +54,11 @@ func newComparingQueryRunner(
}
}

func (nqr ComparingQueryRunner) runQuery(q tools.Query, expectedErrs bool, ast sqlparser.Statement) error {
func (nqr ComparingQueryRunner) runQuery(q data.Query, expectedErrs bool, ast sqlparser.Statement) error {
return nqr.execute(q, expectedErrs, ast)
}

func (nqr *ComparingQueryRunner) execute(query tools.Query, expectedErrs bool, ast sqlparser.Statement) error {
func (nqr *ComparingQueryRunner) execute(query data.Query, expectedErrs bool, ast sqlparser.Statement) error {
if len(query.Query) == 0 {
return nil
}
44 changes: 12 additions & 32 deletions go/tester/tester.go
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ package tester
import (
"encoding/json"
"fmt"
"github.com/vitessio/vitess-tester/go/tools"
"github.com/vitessio/vitess-tester/go/data"
"github.com/vitessio/vitess-tester/go/typ"

"io"
@@ -66,7 +66,7 @@ type (
}

QueryRunner interface {
runQuery(q tools.Query, expectedErrs bool, ast sqlparser.Statement) error
runQuery(q data.Query, expectedErrs bool, ast sqlparser.Statement) error
}

QueryRunnerFactory interface {
@@ -154,37 +154,17 @@ func (t *Tester) Run() error {
if t.autoVSchema() {
defer t.postProcess()
}
data, err := tools.ReadData(t.name)
if err != nil {
return err
}

queries, err := tools.LoadQueries(data)
queries, err := data.LoadQueries(t.name)
if err != nil {
t.reporter.AddFailure(err)
return err
}

for _, q := range queries {
switch q.Type {
// no-ops
case typ.Q_ENABLE_QUERY_LOG,
typ.Q_DISABLE_QUERY_LOG,
typ.Q_ECHO,
typ.Q_DISABLE_WARNINGS,
typ.Q_ENABLE_WARNINGS,
typ.Q_ENABLE_INFO,
typ.Q_DISABLE_INFO,
typ.Q_ENABLE_RESULT_LOG,
typ.Q_DISABLE_RESULT_LOG,
typ.Q_SORTED_RESULT,
typ.Q_REPLACE_REGEX:
// do nothing
case typ.Q_SKIP:
case typ.Skip:
t.skipNext = true
case typ.Q_BEGIN_CONCURRENT, typ.Q_END_CONCURRENT, typ.Q_CONNECT, typ.Q_CONNECTION, typ.Q_DISCONNECT, typ.Q_LET, typ.Q_REPLACE_COLUMN:
t.reporter.AddFailure(fmt.Errorf("%s not supported", q.Type.String()))
case typ.Q_SKIP_IF_BELOW_VERSION:
case typ.SkipIfBelowVersion:
strs := strings.Split(q.Query, " ")
if len(strs) != 3 {
t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.Q_SKIP_IF_BELOW_VERSION in: %v", q.Query))
@@ -197,19 +177,19 @@ func (t *Tester) Run() error {
t.reporter.AddFailure(err)
continue
}
case typ.Q_ERROR:
case typ.Error:
t.expectedErrs = true
case typ.Q_VEXPLAIN:
case typ.VExplain:
strs := strings.Split(q.Query, " ")
if len(strs) != 2 {
t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.Q_VEXPLAIN in: %v", q.Query))
t.reporter.AddFailure(fmt.Errorf("incorrect syntax for typ.VExplain in: %v", q.Query))
continue
}

t.vexplain = strs[1]
case typ.Q_WAIT_FOR_AUTHORITATIVE:
case typ.WaitForAuthoritative:
t.waitAuthoritative(q.Query)
case typ.Q_QUERY:
case typ.Query:
if t.vexplain != "" {
result, err := t.curr.VtConn.ExecuteFetch(fmt.Sprintf("vexplain %s %s", t.vexplain, q.Query), -1, false)
t.vexplain = ""
@@ -221,7 +201,7 @@ func (t *Tester) Run() error {
}

t.runQuery(q)
case typ.Q_REMOVE_FILE:
case typ.RemoveFile:
err = os.Remove(strings.TrimSpace(q.Query))
if err != nil {
return errors.Annotate(err, "failed to remove file")
@@ -235,7 +215,7 @@ func (t *Tester) Run() error {
return nil
}

func (t *Tester) runQuery(q tools.Query) {
func (t *Tester) runQuery(q data.Query) {
if t.skipNext {
t.skipNext = false
return
6 changes: 3 additions & 3 deletions go/tester/tracer.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/vitessio/vitess-tester/go/tools"
"github.com/vitessio/vitess-tester/go/data"
"os"

"vitess.io/vitess/go/mysql"
@@ -67,7 +67,7 @@ func newTracer(traceFile *os.File,
}
}

func (t *Tracer) runQuery(q tools.Query, expectErr bool, ast sqlparser.Statement) error {
func (t *Tracer) runQuery(q data.Query, expectErr bool, ast sqlparser.Statement) error {
if sqlparser.IsDMLStatement(ast) && t.traceFile != nil && !expectErr {
// we don't want to run DMLs twice, so we just run them once while tracing
var errs []error
@@ -100,7 +100,7 @@ func (t *Tracer) runQuery(q tools.Query, expectErr bool, ast sqlparser.Statement
}

// trace writes the query and its trace (fetched from VtConn) as a JSON object into traceFile
func (t *Tracer) trace(query tools.Query) error {
func (t *Tracer) trace(query data.Query) error {
// Marshal the query into JSON format for safe embedding
queryJSON, err := json.Marshal(query.Query)
if err != nil {
220 changes: 18 additions & 202 deletions go/typ/typ.go
Original file line number Diff line number Diff line change
@@ -21,211 +21,27 @@ import "strings"
type CmdType int

const (
Q_CONNECTION CmdType = iota + 1
Q_QUERY
Q_CONNECT
Q_SLEEP
Q_REAL_SLEEP
Q_INC
Q_DEC
Q_SOURCE
Q_DISCONNECT
Q_LET
Q_ECHO
Q_WHILE
Q_END_BLOCK
Q_SYSTEM
Q_RESULT
Q_REQUIRE
Q_SAVE_MASTER_POS
Q_SYNC_WITH_MASTER
Q_SYNC_SLAVE_WITH_MASTER
Q_ERROR
Q_SEND
Q_REAP
Q_DIRTY_CLOSE
Q_REPLACE
Q_REPLACE_COLUMN
Q_PING
Q_EVAL
Q_EVAL_RESULT
Q_ENABLE_QUERY_LOG
Q_DISABLE_QUERY_LOG
Q_ENABLE_RESULT_LOG
Q_DISABLE_RESULT_LOG
Q_ENABLE_CONNECT_LOG
Q_DISABLE_CONNECT_LOG
Q_WAIT_FOR_SLAVE_TO_STOP
Q_ENABLE_WARNINGS
Q_DISABLE_WARNINGS
Q_ENABLE_INFO
Q_DISABLE_INFO
Q_ENABLE_SESSION_TRACK_INFO
Q_DISABLE_SESSION_TRACK_INFO
Q_ENABLE_METADATA
Q_DISABLE_METADATA
Q_EXEC
Q_EXECW
Q_DELIMITER
Q_DISABLE_ABORT_ON_ERROR
Q_ENABLE_ABORT_ON_ERROR
Q_DISPLAY_VERTICAL_RESULTS
Q_DISPLAY_HORIZONTAL_RESULTS
Q_QUERY_VERTICAL
Q_QUERY_HORIZONTAL
Q_SORTED_RESULT
Q_LOWERCASE
Q_START_TIMER
Q_END_TIMER
Q_CHARACTER_SET
Q_DISABLE_PS_PROTOCOL
Q_ENABLE_PS_PROTOCOL
Q_DISABLE_RECONNECT
Q_ENABLE_RECONNECT
Q_IF
Q_DISABLE_PARSING
Q_ENABLE_PARSING
Q_REPLACE_REGEX
Q_REPLACE_NUMERIC_ROUND
Q_REMOVE_FILE
Q_FILE_EXIST
Q_WRITE_FILE
Q_COPY_FILE
Q_PERL
Q_DIE
Q_EXIT
Q_SKIP
Q_CHMOD_FILE
Q_APPEND_FILE
Q_CAT_FILE
Q_DIFF_FILES
Q_SEND_QUIT
Q_CHANGE_USER
Q_MKDIR
Q_RMDIR
Q_LIST_FILES
Q_LIST_FILES_WRITE_FILE
Q_LIST_FILES_APPEND_FILE
Q_SEND_SHUTDOWN
Q_SHUTDOWN_SERVER
Q_RESULT_FORMAT_VERSION
Q_MOVE_FILE
Q_REMOVE_FILES_WILDCARD
Q_SEND_EVAL
Q_OUTPUT /* redirect output to a file */
Q_RESET_CONNECTION
Q_SINGLE_QUERY
Q_BEGIN_CONCURRENT
Q_END_CONCURRENT
Q_UNKNOWN
Q_COMMENT
Q_COMMENT_WITH_COMMAND
Q_EMPTY_LINE
Q_SKIP_IF_BELOW_VERSION
Q_VEXPLAIN
Q_WAIT_FOR_AUTHORITATIVE
Query CmdType = iota
Error
RemoveFile
Skip
Unknown
Comment
CommentWithCommand
EmptyLine
SkipIfBelowVersion
VExplain
WaitForAuthoritative
)

var commandMap = map[string]CmdType{
"connection": Q_CONNECTION,
"query": Q_QUERY,
"connect": Q_CONNECT,
"sleep": Q_SLEEP,
"real_sleep": Q_REAL_SLEEP,
"inc": Q_INC,
"dec": Q_DEC,
"source": Q_SOURCE,
"disconnect": Q_DISCONNECT,
"let": Q_LET,
"echo": Q_ECHO,
"while": Q_WHILE,
"end": Q_END_BLOCK,
"system": Q_SYSTEM,
"result": Q_RESULT,
"require": Q_REQUIRE,
"save_master_pos": Q_SAVE_MASTER_POS,
"sync_with_master": Q_SYNC_WITH_MASTER,
"sync_slave_with_master": Q_SYNC_SLAVE_WITH_MASTER,
"error": Q_ERROR,
"send": Q_SEND,
"reap": Q_REAP,
"dirty_close": Q_DIRTY_CLOSE,
"replace_result": Q_REPLACE,
"replace_column": Q_REPLACE_COLUMN,
"ping": Q_PING,
"eval": Q_EVAL,
"eval_result": Q_EVAL_RESULT,
"enable_query_log": Q_ENABLE_QUERY_LOG,
"disable_query_log": Q_DISABLE_QUERY_LOG,
"enable_result_log": Q_ENABLE_RESULT_LOG,
"disable_result_log": Q_DISABLE_RESULT_LOG,
"enable_connect_log": Q_ENABLE_CONNECT_LOG,
"disable_connect_log": Q_DISABLE_CONNECT_LOG,
"wait_for_slave_to_stop": Q_WAIT_FOR_SLAVE_TO_STOP,
"enable_warnings": Q_ENABLE_WARNINGS,
"disable_warnings": Q_DISABLE_WARNINGS,
"enable_info": Q_ENABLE_INFO,
"disable_info": Q_DISABLE_INFO,
"enable_session_track_info": Q_ENABLE_SESSION_TRACK_INFO,
"disable_session_track_info": Q_DISABLE_SESSION_TRACK_INFO,
"enable_metadata": Q_ENABLE_METADATA,
"disable_metadata": Q_DISABLE_METADATA,
"exec": Q_EXEC,
"execw": Q_EXECW,
"delimiter": Q_DELIMITER,
"disable_abort_on_error": Q_DISABLE_ABORT_ON_ERROR,
"enable_abort_on_error": Q_ENABLE_ABORT_ON_ERROR,
"vertical_results": Q_DISPLAY_VERTICAL_RESULTS,
"horizontal_results": Q_DISPLAY_HORIZONTAL_RESULTS,
"query_vertical": Q_QUERY_VERTICAL,
"query_horizontal": Q_QUERY_HORIZONTAL,
"sorted_result": Q_SORTED_RESULT,
"lowercase_result": Q_LOWERCASE,
"start_timer": Q_START_TIMER,
"end_timer": Q_END_TIMER,
"character_set": Q_CHARACTER_SET,
"disable_ps_protocol": Q_DISABLE_PS_PROTOCOL,
"enable_ps_protocol": Q_ENABLE_PS_PROTOCOL,
"disable_reconnect": Q_DISABLE_RECONNECT,
"enable_reconnect": Q_ENABLE_RECONNECT,
"if": Q_IF,
"disable_parsing": Q_DISABLE_PARSING,
"enable_parsing": Q_ENABLE_PARSING,
"replace_regex": Q_REPLACE_REGEX,
"replace_numeric_round": Q_REPLACE_NUMERIC_ROUND,
"remove_file": Q_REMOVE_FILE,
"file_exists": Q_FILE_EXIST,
"write_file": Q_WRITE_FILE,
"copy_file": Q_COPY_FILE,
"perl": Q_PERL,
"die": Q_DIE,
"exit": Q_EXIT,
"skip": Q_SKIP,
"chmod": Q_CHMOD_FILE,
"append_file": Q_APPEND_FILE,
"cat_file": Q_CAT_FILE,
"diff_files": Q_DIFF_FILES,
"send_quit": Q_SEND_QUIT,
"change_user": Q_CHANGE_USER,
"mkdir": Q_MKDIR,
"rmdir": Q_RMDIR,
"list_files": Q_LIST_FILES,
"list_files_write_file": Q_LIST_FILES_WRITE_FILE,
"list_files_append_file": Q_LIST_FILES_APPEND_FILE,
"send_shutdown": Q_SEND_SHUTDOWN,
"shutdown_server": Q_SHUTDOWN_SERVER,
"result_format": Q_RESULT_FORMAT_VERSION,
"move_file": Q_MOVE_FILE,
"remove_files_wildcard": Q_REMOVE_FILES_WILDCARD,
"send_eval": Q_SEND_EVAL,
"output": Q_OUTPUT,
"reset_connection": Q_RESET_CONNECTION,
"single_query": Q_SINGLE_QUERY,
"begin_concurrent": Q_BEGIN_CONCURRENT,
"end_concurrent": Q_END_CONCURRENT,
"skip_if_below_version": Q_SKIP_IF_BELOW_VERSION,
"vexplain": Q_VEXPLAIN,
"wait_authoritative": Q_WAIT_FOR_AUTHORITATIVE,
"query": Query,
"error": Error,
"remove_file": RemoveFile,
"skip": Skip,
"skip_if_below_version": SkipIfBelowVersion,
"vexplain": VExplain,
"wait_authoritative": WaitForAuthoritative,
}

func (cmd CmdType) String() string {

0 comments on commit 33030b3

Please sign in to comment.