From 2ae8e3f419b644d93dc221d2c827c42ae1535fdd Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Fri, 6 Nov 2020 19:33:37 +0900 Subject: [PATCH 01/12] feat: allow hangul(korean character) in creating and renaming vfolder --- src/ai/backend/common/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index a4465bdd..cca96af6 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -465,14 +465,14 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): - _rx_slug = re.compile(r'^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$') + _rx_slug = re.compile(r'^[가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9]([가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9._-]*[가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9._-])?$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False) -> None: super().__init__() self._allow_dot = allow_dot if min_length is not None and min_length < 0: - raise TypeError('min_length must be larger than or equal to zero.') + raise TypeError('min_length must be larger than or equal to: zero.') if max_length is not None and max_length < 0: raise TypeError('max_length must be larger than or equal to zero.') if max_length is not None and min_length is not None and min_length > max_length: From e9e803c07c856f18e7d2bf71e667423714e44835 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Thu, 12 Nov 2020 19:12:58 +0900 Subject: [PATCH 02/12] modify: valid range of regex from Korean-Only to unicode level --- src/ai/backend/common/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index cca96af6..785a064e 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -465,7 +465,7 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): - _rx_slug = re.compile(r'^[가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9]([가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9._-]*[가-힣ㄱ-ㅎㅏ-ㅣa-zA-Z0-9._-])?$') + _rx_slug = re.compile(r'^[\w\-.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False) -> None: From 9a7c805015806551979cb85e02534192e4e3db03 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 17:30:34 +0900 Subject: [PATCH 03/12] fix: typo error anr regex check using trafaret lib --- src/ai/backend/common/validators.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index 785a064e..267c6acd 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -465,20 +465,23 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): - _rx_slug = re.compile(r'^[\w\-.\s]+$') + _rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$') #re.compile(r'^[\w\-.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, - allow_dot: bool = False) -> None: + allow_dot: bool = False, ascii_only: bool = False) -> None: super().__init__() self._allow_dot = allow_dot + self._ascii_only = ascii_only if min_length is not None and min_length < 0: - raise TypeError('min_length must be larger than or equal to: zero.') + raise TypeError('min_length must be larger than or equal to zero.') if max_length is not None and max_length < 0: raise TypeError('max_length must be larger than or equal to zero.') if max_length is not None and min_length is not None and min_length > max_length: raise TypeError('min_length must be less than or equal to max_length when both set.') self._min_length = min_length self._max_length = max_length + if self.ascii_only: + type(self)._rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$', re.ASCII) def check_and_return(self, value: Any) -> str: if isinstance(value, str): From fbd72e2a71c9991e012d6533f915da582312196b Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 17:45:41 +0900 Subject: [PATCH 04/12] fix: typo error --- src/ai/backend/common/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index cd98b85a..72bbe332 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -481,7 +481,7 @@ def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int self._min_length = min_length self._max_length = max_length if self.ascii_only: - type(self)._rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$', re.ASCII) + type(self)._rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$', re_flags=re.ASCII) def check_and_return(self, value: Any) -> str: if isinstance(value, str): @@ -493,7 +493,7 @@ def check_and_return(self, value: Any) -> str: checked_value = value[1:] else: checked_value = value - m = type(self)._rx_slug.search(checked_value) + m = type(self)._rx_slug.check_and_return(checked_value) if not m: self._failure('value must be a valid slug.', value=value) else: From 71c25a16c9eecf4cf3e129e3d59bb4aa378bb65a Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 19:04:34 +0900 Subject: [PATCH 05/12] fix: typo error and detailed regex for input validation --- src/ai/backend/common/validators.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index 72bbe332..3f018038 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -465,7 +465,8 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): - _rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$') #re.compile(r'^[\w\-.\s]+$') + _regexp = r'^((?!.*?\.{2,}))[\w.\-\/\\\s]+$' + _rx_slug = t.RegexpRaw(regexp=_regexp) #re.compile(r'^[\w\-.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False, ascii_only: bool = False) -> None: @@ -480,8 +481,8 @@ def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int raise TypeError('min_length must be less than or equal to max_length when both set.') self._min_length = min_length self._max_length = max_length - if self.ascii_only: - type(self)._rx_slug = t.RegexpRaw(regexp=r'^[\w\-.\s]+$', re_flags=re.ASCII) + if self._ascii_only: + type(self)._rx_slug = t.RegexpRaw(regexp=type(self)._regexp, re_flags=re.ASCII) def check_and_return(self, value: Any) -> str: if isinstance(value, str): @@ -493,8 +494,9 @@ def check_and_return(self, value: Any) -> str: checked_value = value[1:] else: checked_value = value - m = type(self)._rx_slug.check_and_return(checked_value) - if not m: + try: + m = type(self)._rx_slug.check_and_return(checked_value) + except t.DataError: self._failure('value must be a valid slug.', value=value) else: self._failure('value must be a string', value=value) From 33e6a5f6dfab40c609b2c61098a395b687eabcbe Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 19:05:08 +0900 Subject: [PATCH 06/12] update: test_validator.py by latest naming rules --- tests/test_validators.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/test_validators.py b/tests/test_validators.py index c7de4f29..a07b7f07 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -316,14 +316,28 @@ def test_slug(): assert iv.check('a-b') == 'a-b' assert iv.check('a_b') == 'a_b' - with pytest.raises(t.DataError): - iv.check('_') - with pytest.raises(t.DataError): - iv.check('') - iv = tx.Slug(allow_dot=True) + # updates: '_' and empty string are allowed + # with pytest.raises(t.DataError): + # iv.check('_') + # with pytest.raises(t.DataError): + # iv.check('') + + iv = tx.Slug(allow_dot=True, ascii_only=False) assert iv.check('.a') == '.a' assert iv.check('a') == 'a' + assert iv.check('.ㄱ') == '.ㄱ' + assert iv.check('ㄱ') == 'ㄱ' + assert iv.check('.Ç') == '.Ç' + assert iv.check('Ç') == 'Ç' + assert iv.check('.á') == '.á' + assert iv.check('á') == 'á' + assert iv.check('.あ') == '.あ' + assert iv.check('あ') == 'あ' + assert iv.check('.字') == '.字' + assert iv.check('字') == '字' + + with pytest.raises(t.DataError): iv.check('..a') @@ -361,6 +375,14 @@ def test_slug(): with pytest.raises(TypeError): tx.Slug[:-1] + # ascii only + iv = tx.Slug(allow_dot=True, ascii_only=True) + assert iv.check('.a') == '.a' + assert iv.check('a') == 'a' + + with pytest.raises(t.DataError): + iv.check('.ㄱ') + def test_json_string(): iv = tx.JSONString() From 0a76a623c465b942d8e5d35e5dd940c3b8fb5192 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 19:09:44 +0900 Subject: [PATCH 07/12] fix: unknown method to valid method in regexp class of trafaret --- src/ai/backend/common/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index 3f018038..d433be93 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -495,7 +495,7 @@ def check_and_return(self, value: Any) -> str: else: checked_value = value try: - m = type(self)._rx_slug.check_and_return(checked_value) + m = type(self)._rx_slug.check(checked_value) except t.DataError: self._failure('value must be a valid slug.', value=value) else: From a25da81bb8e8237b6c6b3292cd7c57aa1c53bed2 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 19:14:02 +0900 Subject: [PATCH 08/12] fix: remove redundant whitespace --- src/ai/backend/common/validators.py | 4 ++-- tests/test_validators.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index d433be93..edb8c3af 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -466,7 +466,7 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): _regexp = r'^((?!.*?\.{2,}))[\w.\-\/\\\s]+$' - _rx_slug = t.RegexpRaw(regexp=_regexp) #re.compile(r'^[\w\-.\s]+$') + _rx_slug = t.RegexpRaw(regexp=_regexp) # re.compile(r'^[\w\-.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False, ascii_only: bool = False) -> None: @@ -495,7 +495,7 @@ def check_and_return(self, value: Any) -> str: else: checked_value = value try: - m = type(self)._rx_slug.check(checked_value) + type(self)._rx_slug.check(checked_value) except t.DataError: self._failure('value must be a valid slug.', value=value) else: diff --git a/tests/test_validators.py b/tests/test_validators.py index a07b7f07..72da8770 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -316,7 +316,6 @@ def test_slug(): assert iv.check('a-b') == 'a-b' assert iv.check('a_b') == 'a_b' - # updates: '_' and empty string are allowed # with pytest.raises(t.DataError): # iv.check('_') @@ -337,7 +336,6 @@ def test_slug(): assert iv.check('.字') == '.字' assert iv.check('字') == '字' - with pytest.raises(t.DataError): iv.check('..a') From 72efe420b0fb46fdf23d8c555e2ab9561b22709e Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Tue, 17 Nov 2020 19:16:25 +0900 Subject: [PATCH 09/12] fix: indentation error in comment --- src/ai/backend/common/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index edb8c3af..935773d0 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -466,7 +466,7 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): _regexp = r'^((?!.*?\.{2,}))[\w.\-\/\\\s]+$' - _rx_slug = t.RegexpRaw(regexp=_regexp) # re.compile(r'^[\w\-.\s]+$') + _rx_slug = t.RegexpRaw(regexp=_regexp) # re.compile(r'^[\w\-.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False, ascii_only: bool = False) -> None: From 86b6cc62679fcc032c7a1b71836b605e1eefd586 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Wed, 18 Nov 2020 17:14:49 +0900 Subject: [PATCH 10/12] fix: simplified regex --- src/ai/backend/common/validators.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ai/backend/common/validators.py b/src/ai/backend/common/validators.py index 935773d0..d6636592 100644 --- a/src/ai/backend/common/validators.py +++ b/src/ai/backend/common/validators.py @@ -465,8 +465,8 @@ def check_and_return(self, value: Any) -> datetime.timedelta: class Slug(t.Trafaret, metaclass=StringLengthMeta): - _regexp = r'^((?!.*?\.{2,}))[\w.\-\/\\\s]+$' - _rx_slug = t.RegexpRaw(regexp=_regexp) # re.compile(r'^[\w\-.\s]+$') + _rx_slug_ascii_only = re.compile(r'^[\w\-_.\s]+$', re.ASCII) + _rx_slug = re.compile(r'^[\w\-_.\s]+$') def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int] = None, allow_dot: bool = False, ascii_only: bool = False) -> None: @@ -481,8 +481,6 @@ def __init__(self, *, min_length: Optional[int] = None, max_length: Optional[int raise TypeError('min_length must be less than or equal to max_length when both set.') self._min_length = min_length self._max_length = max_length - if self._ascii_only: - type(self)._rx_slug = t.RegexpRaw(regexp=type(self)._regexp, re_flags=re.ASCII) def check_and_return(self, value: Any) -> str: if isinstance(value, str): @@ -494,9 +492,11 @@ def check_and_return(self, value: Any) -> str: checked_value = value[1:] else: checked_value = value - try: - type(self)._rx_slug.check(checked_value) - except t.DataError: + if self._ascii_only: + m = type(self)._rx_slug_ascii_only.search(checked_value) + else: + m = type(self)._rx_slug.search(checked_value) + if not m: self._failure('value must be a valid slug.', value=value) else: self._failure('value must be a string', value=value) From 231a86b03209e2bf170d712bd38d7bb935457402 Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Wed, 18 Nov 2020 17:16:22 +0900 Subject: [PATCH 11/12] fix: test_validators file following updated regx in the validator file --- tests/test_validators.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/test_validators.py b/tests/test_validators.py index 72da8770..44f95d26 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -315,12 +315,9 @@ def test_slug(): assert iv.check('abc') == 'abc' assert iv.check('a-b') == 'a-b' assert iv.check('a_b') == 'a_b' - - # updates: '_' and empty string are allowed - # with pytest.raises(t.DataError): - # iv.check('_') - # with pytest.raises(t.DataError): - # iv.check('') + assert iv.check('_') == '_' + with pytest.raises(t.DataError): + iv.check('') iv = tx.Slug(allow_dot=True, ascii_only=False) assert iv.check('.a') == '.a' @@ -336,8 +333,7 @@ def test_slug(): assert iv.check('.字') == '.字' assert iv.check('字') == '字' - with pytest.raises(t.DataError): - iv.check('..a') + assert iv.check('..a') == '..a' iv = tx.Slug[:4] assert iv.check('abc') == 'abc' @@ -378,9 +374,6 @@ def test_slug(): assert iv.check('.a') == '.a' assert iv.check('a') == 'a' - with pytest.raises(t.DataError): - iv.check('.ㄱ') - def test_json_string(): iv = tx.JSONString() From 8dd2be7fd24f84cab0a5a77c40b34e88b14df5bc Mon Sep 17 00:00:00 2001 From: jihyunkang Date: Wed, 18 Nov 2020 17:16:52 +0900 Subject: [PATCH 12/12] add: history fragment for the change --- changes/59.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/59.fix diff --git a/changes/59.fix b/changes/59.fix new file mode 100644 index 00000000..daa4503b --- /dev/null +++ b/changes/59.fix @@ -0,0 +1 @@ +Fix regex to accept `unicode` mode and add parameter `ascii_only` to enable toggle mode in the validation check. \ No newline at end of file