From 81dff77f4df293ca31e9ace51096918669f15223 Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:25:47 +0800 Subject: [PATCH 1/3] fix: regular expressions not matching multiple lines --- sqle/utils/util.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index a5ed2211e..cc8974202 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -318,9 +318,20 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool { return true } -// 全模糊匹配字符串，并且对大小写不敏感 +// 全模糊匹配字符串，对大小写不敏感，匹配多行，且防止正则注入 func FullFuzzySearchRegexp(str string) *regexp.Regexp { - return regexp.MustCompile(`^.*(?i)` + regexp.QuoteMeta(str) + `.*$`) + /* + 1. (?is)是一个正则表达式修饰符,其中： + i表示忽略大小写(case-insensitive) + s表示让.匹配任何字符,包括换行符(\n) + 2. ^.*匹配字符串的开头,其中： + ^表示起始位置, + .表示匹配任何字符(除了换行符) + *表示匹配前面的模式零次或多次 + 3. .*$匹配字符串的结尾,其中： + $表示结束位置 + */ + return regexp.MustCompile(`(?is)^.*` + regexp.QuoteMeta(str) + `.*$`) } var ErrUnknownEncoding = errors.New("unknown encoding") From 0cdefc48515cea6be585e688ad0b923af50d67ad Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:26:42 +0800 Subject: [PATCH 2/3] test: unit test for regular expressions and black list --- sqle/api/controller/v1/audit_plan_test.go | 8 ++++++++ sqle/utils/util_test.go | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/sqle/api/controller/v1/audit_plan_test.go b/sqle/api/controller/v1/audit_plan_test.go index 280bfc9d6..f6f13ca49 100644 --- a/sqle/api/controller/v1/audit_plan_test.go +++ b/sqle/api/controller/v1/audit_plan_test.go @@ -15,6 +15,9 @@ func TestIsSqlInBlackList(t *testing.T) { }, { FilterContent: "table_1", FilterType: "SQL", + },{ + FilterContent: "ignored_service", + FilterType: "SQL", }, }) @@ -22,6 +25,9 @@ func TestIsSqlInBlackList(t *testing.T) { "SELECT * FROM users", "DELETE From tAble_1", "SELECT COUNT(*) FROM table_2", + `/* this is a comment, Service: ignored_service */ + select * from table_ignored where id < 123;`, + `/* this is a comment, Service: ignored_service */ update * from table_ignored where id < 123;`, } for _, matchSql := range matchSqls { if !filter.IsSqlInBlackList(matchSql) { @@ -32,6 +38,8 @@ func TestIsSqlInBlackList(t *testing.T) { "INSERT INTO users VALUES (1, 'John')", "DELETE From schools", "SHOW CREATE TABLE table_2", + `/* this is a comment, Service: ignored_ + service */ update * from table_ignored where id < 123;`, } for _, notMatchSql := range notMatchSqls { if filter.IsSqlInBlackList(notMatchSql) { diff --git a/sqle/utils/util_test.go b/sqle/utils/util_test.go index d9da4c578..92d5690f2 100644 --- a/sqle/utils/util_test.go +++ b/sqle/utils/util_test.go @@ -261,6 +261,13 @@ func TestFullFuzzySearchRegexp(t *testing.T) { ".*(?i)", []string{"GoLang .*(?i) awesome", "I love GO^.*(?i)SING", "GoLangGO.*(?i)Golang"}, []string{"language", "hi", "heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, + },{ + "ignored_service", + []string{`/* this is a comment, Service: ignored_service */ + select * from table_ignored where id < 123;' + `,`/* this is a comment, Service: ignored_service */ select * from table_ignored where id < 123;`}, + []string{"any sql","",`/* this is a comment, Service: ignored + _service */ select * from table_ignored where id < 123;`}, }, } From c22c755bcd9609bf419a6720e74ff57575800feb Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:50:30 +0800 Subject: [PATCH 3/3] docs: modify comment --- sqle/utils/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index cc8974202..3026b5adf 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -323,7 +323,7 @@ func FullFuzzySearchRegexp(str string) *regexp.Regexp { /* 1. (?is)是一个正则表达式修饰符,其中： i表示忽略大小写(case-insensitive) - s表示让.匹配任何字符,包括换行符(\n) + s表示开启单行模式，开启后.可以匹配换行符，让整个字符串作为一行 2. ^.*匹配字符串的开头,其中： ^表示起始位置, .表示匹配任何字符(除了换行符)