From 8ffe5689664b6393c6b8d8217e74e6d187ea7186 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 1 Jun 2021 11:53:53 +0800 Subject: [PATCH 01/15] add "&&" as KW_AND Note: WIP --- zetasql/parser/flex_tokenizer.l | 1 + zetasql/parser/testdata/in.test | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index 959e43698..acba99a78 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -357,6 +357,7 @@ and { } return BisonParserImpl::token::KW_AND; } +"&&" { return BisonParserImpl::token::KW_AND; } anonymization { return BisonParserImpl::token::KW_ANONYMIZATION; } any { return BisonParserImpl::token::KW_ANY; } array { return BisonParserImpl::token::KW_ARRAY; } diff --git a/zetasql/parser/testdata/in.test b/zetasql/parser/testdata/in.test index fec8ab28c..538affbc7 100644 --- a/zetasql/parser/testdata/in.test +++ b/zetasql/parser/testdata/in.test @@ -159,6 +159,39 @@ SELECT col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) == +select col IN ('a') && col NOT BETWEEN x + y AND f(x,y) +-- +QueryStatement [0-55] + Query [0-55] + Select [0-55] + SelectList [7-55] + SelectColumn [7-55] + AndExpr [7-55] + InExpression(IN) [11-19] + PathExpression [7-10] + Identifier(col) [7-10] + InList [15-18] + StringLiteral('a') [15-18] + BetweenExpression(NOT BETWEEN) [31-55] + PathExpression [23-26] + Identifier(col) [23-26] + BinaryExpression(+) [39-44] + PathExpression [39-40] + Identifier(x) [39-40] + PathExpression [43-44] + Identifier(y) [43-44] + FunctionCall [49-55] + PathExpression [49-50] + Identifier(f) [49-50] + PathExpression [51-52] + Identifier(x) [51-52] + PathExpression [53-54] + Identifier(y) [53-54] +-- +SELECT + col IN ('a') && col NOT BETWEEN x + y AND f(x, y) +== + select * from T where @Id IN (@ValidIds) -- QueryStatement [0-40] From 703f28645ba043b3d6105b7683e637a050813200 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 1 Jun 2021 12:30:26 +0800 Subject: [PATCH 02/15] fix `&&` test --- zetasql/parser/testdata/in.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetasql/parser/testdata/in.test b/zetasql/parser/testdata/in.test index 538affbc7..8ff3e4f35 100644 --- a/zetasql/parser/testdata/in.test +++ b/zetasql/parser/testdata/in.test @@ -189,7 +189,7 @@ QueryStatement [0-55] Identifier(y) [53-54] -- SELECT - col IN ('a') && col NOT BETWEEN x + y AND f(x, y) + col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) == select * from T where @Id IN (@ValidIds) From 17c6b02aa9941437d561605911cf702279e54de7 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 1 Jun 2021 17:37:43 +0800 Subject: [PATCH 03/15] add "!" as a synonym to "NOT" --- zetasql/parser/flex_tokenizer.l | 11 +++++- .../parser/testdata/bitwise_operators.test | 2 +- zetasql/parser/testdata/in.test | 38 +++++++++---------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index acba99a78..974448ec9 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -276,6 +276,8 @@ pound_comment #[^\r\n]*(\r|\n|\r\n)? comment ({cs_comment}|{dash_comment}|{pound_comment}) +not (not|"!") + %% /* RULES SECTION @@ -518,7 +520,7 @@ module { return BisonParserImpl::token::KW_MODULE; } natural { return BisonParserImpl::token::KW_NATURAL; } new { return BisonParserImpl::token::KW_NEW; } no { return BisonParserImpl::token::KW_NO; } -not { return BisonParserImpl::token::KW_NOT; } +{not} { return BisonParserImpl::token::KW_NOT; } /* This returns a different token because returning KW_NOT would confuse the operator precedence parsing. Boolean NOT has a different precedence than NOT BETWEEN/IN/LIKE/DISTINCT. The final character at the end is intended to @@ -534,6 +536,13 @@ not{whitespace}(between|in|like|distinct)[^A-Z_0-9] { } return BisonParserImpl::token::KW_NOT_SPECIAL; } +"!"{whitespace}(between|in|like|distinct)[^A-Z_0-9] { + SET_RETURN_PREFIX_LENGTH(1); + if (mode_ == BisonParserMode::kTokenizer) { + return BisonParserImpl::token::KW_NOT; + } + return BisonParserImpl::token::KW_NOT_SPECIAL; +} null { return BisonParserImpl::token::KW_NULL; } nulls { return BisonParserImpl::token::KW_NULLS; } numeric { return BisonParserImpl::token::KW_NUMERIC; } diff --git a/zetasql/parser/testdata/bitwise_operators.test b/zetasql/parser/testdata/bitwise_operators.test index 74aac34aa..5751a2f24 100644 --- a/zetasql/parser/testdata/bitwise_operators.test +++ b/zetasql/parser/testdata/bitwise_operators.test @@ -66,7 +66,7 @@ select 1 < > 2 -- ALTERNATION GROUP: ! = -- -ERROR: Syntax error: Illegal input character "!" [at 1:10] +ERROR: Syntax error: Expected end of input but got keyword NOT [at 1:10] select 1 ! = 2 ^ == diff --git a/zetasql/parser/testdata/in.test b/zetasql/parser/testdata/in.test index 8ff3e4f35..78b447549 100644 --- a/zetasql/parser/testdata/in.test +++ b/zetasql/parser/testdata/in.test @@ -159,34 +159,34 @@ SELECT col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) == -select col IN ('a') && col NOT BETWEEN x + y AND f(x,y) --- -QueryStatement [0-55] - Query [0-55] - Select [0-55] - SelectList [7-55] - SelectColumn [7-55] - AndExpr [7-55] +select col IN ('a') && col ! BETWEEN x + y AND f(x,y) +-- +QueryStatement [0-53] + Query [0-53] + Select [0-53] + SelectList [7-53] + SelectColumn [7-53] + AndExpr [7-53] InExpression(IN) [11-19] PathExpression [7-10] Identifier(col) [7-10] InList [15-18] StringLiteral('a') [15-18] - BetweenExpression(NOT BETWEEN) [31-55] + BetweenExpression(NOT BETWEEN) [29-53] PathExpression [23-26] Identifier(col) [23-26] - BinaryExpression(+) [39-44] - PathExpression [39-40] - Identifier(x) [39-40] - PathExpression [43-44] - Identifier(y) [43-44] - FunctionCall [49-55] + BinaryExpression(+) [37-42] + PathExpression [37-38] + Identifier(x) [37-38] + PathExpression [41-42] + Identifier(y) [41-42] + FunctionCall [47-53] + PathExpression [47-48] + Identifier(f) [47-48] PathExpression [49-50] - Identifier(f) [49-50] + Identifier(x) [49-50] PathExpression [51-52] - Identifier(x) [51-52] - PathExpression [53-54] - Identifier(y) [53-54] + Identifier(y) [51-52] -- SELECT col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) From a47ff4c6d41af8ecb50da942c46b99760c6ca60c Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 1 Jun 2021 19:02:20 +0800 Subject: [PATCH 04/15] add `div` & `mod` operator --- zetasql/parser/bison_parser.y | 7 ++++--- zetasql/parser/flex_tokenizer.l | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index 779a5fd57..25f804949 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -521,7 +521,8 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %token KW_CONCAT_OP "||" %token '+' "+" %token '-' "-" -%token '/' "/" +%token KW_DIVIDE "/" +%token KW_MOD "%" %token '~' "~" %token '.' "." %token KW_DOT_STAR ".*" @@ -560,7 +561,7 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %left "<<" ">>" %left "+" "-" %left "||" -%left "*" "/" +%left "*" "/" "%" %left UNARY_PRECEDENCE // For all unary operators %precedence DOUBLE_AT_PRECEDENCE // Needs to appear before "." %left PRIMARY_PRECEDENCE "(" ")" "[" "]" "." // For ., .(...), [], etc. @@ -5224,7 +5225,7 @@ additive_operator: multiplicative_operator: "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } - | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } + | KW_DIVIDE { $$ = zetasql::ASTBinaryExpression::DIVIDE; } ; // Returns ShiftOperator to indicate the operator type. diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index 974448ec9..dc09f1def 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -278,6 +278,8 @@ comment ({cs_comment}|{dash_comment}|{pound_comment}) not (not|"!") +mod (mod|"%") + %% /* RULES SECTION @@ -877,7 +879,8 @@ zone { return BisonParserImpl::token::KW_ZONE; } "&" { return '&'; } "+" { return '+'; } "-" { return '-'; } -"/" { return '/'; } +("/"|div) { return BisonParserImpl::token::KW_DIVIDE; } +{mod} { return BisonParserImpl::token::KW_MOD; } "~" { return '~'; } "?" { return '?'; } "@"{opt_whitespace}"{" { From 6702e236c6c65411f6e3b75f6ccef3171d5f176b Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Wed, 2 Jun 2021 00:37:29 +0800 Subject: [PATCH 05/15] div as reversed keyword --- zetasql/parser/bison_parser.y | 13 ++++++++----- zetasql/parser/flex_tokenizer.l | 7 +++---- zetasql/parser/keywords.cc | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index 25f804949..fa2ecb254 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -521,8 +521,8 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %token KW_CONCAT_OP "||" %token '+' "+" %token '-' "-" -%token KW_DIVIDE "/" -%token KW_MOD "%" +%token '/' "/" +%token '%' "%" %token '~' "~" %token '.' "." %token KW_DOT_STAR ".*" @@ -561,7 +561,7 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %left "<<" ">>" %left "+" "-" %left "||" -%left "*" "/" "%" +%left "*" "/" "DIV" "%" %left UNARY_PRECEDENCE // For all unary operators %precedence DOUBLE_AT_PRECEDENCE // Needs to appear before "." %left PRIMARY_PRECEDENCE "(" ")" "[" "]" "." // For ., .(...), [], etc. @@ -672,6 +672,7 @@ using zetasql::ASTDropStatement; %token KW_DEFINE "DEFINE" %token KW_DESC "DESC" %token KW_DISTINCT "DISTINCT" +%token KW_DIVIDE "DIV" %token KW_ELSE "ELSE" %token KW_END "END" %token KW_ENUM "ENUM" @@ -5224,8 +5225,9 @@ additive_operator: ; multiplicative_operator: - "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } - | KW_DIVIDE { $$ = zetasql::ASTBinaryExpression::DIVIDE; } + "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } %prec "*" + | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } %prec "/" + | "DIV" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } ; // Returns ShiftOperator to indicate the operator type. @@ -7075,6 +7077,7 @@ reserved_keyword_rule: | "DEFINE" | "DESC" | "DISTINCT" + | "DIV" | "ELSE" | "END" | "ENUM" diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index dc09f1def..3fd9d121f 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -278,8 +278,6 @@ comment ({cs_comment}|{dash_comment}|{pound_comment}) not (not|"!") -mod (mod|"%") - %% /* RULES SECTION @@ -879,8 +877,9 @@ zone { return BisonParserImpl::token::KW_ZONE; } "&" { return '&'; } "+" { return '+'; } "-" { return '-'; } -("/"|div) { return BisonParserImpl::token::KW_DIVIDE; } -{mod} { return BisonParserImpl::token::KW_MOD; } +"/" { return '/'; } +div { return BisonParserImpl::token::KW_DIVIDE; } +"%" { return '%'; } "~" { return '~'; } "?" { return '?'; } "@"{opt_whitespace}"{" { diff --git a/zetasql/parser/keywords.cc b/zetasql/parser/keywords.cc index e84a56ebc..3556c9f3b 100644 --- a/zetasql/parser/keywords.cc +++ b/zetasql/parser/keywords.cc @@ -111,6 +111,7 @@ constexpr KeywordInfoPOD kAllKeywords[] = { {"descriptor", KW_DESCRIPTOR}, {"deterministic", KW_DETERMINISTIC}, {"distinct", KW_DISTINCT, KeywordInfo::kReserved}, + {"div", KW_DIVIDE, KeywordInfo::kReserved}, {"do", KW_DO}, {"drop", KW_DROP}, {"else", KW_ELSE, KeywordInfo::kReserved}, From 433f1331a72e8d887cff44ceb3f447138a2a9407 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Wed, 2 Jun 2021 11:40:58 +0800 Subject: [PATCH 06/15] add div test --- zetasql/parser/bison_parser.y | 6 ++-- zetasql/parser/flex_tokenizer.l | 4 +-- .../parser/testdata/operator_precedence.test | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index fa2ecb254..38611825a 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -561,7 +561,7 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %left "<<" ">>" %left "+" "-" %left "||" -%left "*" "/" "DIV" "%" +%left "*" "/" "DIV" "%" %left UNARY_PRECEDENCE // For all unary operators %precedence DOUBLE_AT_PRECEDENCE // Needs to appear before "." %left PRIMARY_PRECEDENCE "(" ")" "[" "]" "." // For ., .(...), [], etc. @@ -5225,8 +5225,8 @@ additive_operator: ; multiplicative_operator: - "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } %prec "*" - | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } %prec "/" + "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } + | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } | "DIV" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } ; diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index 3fd9d121f..a854d44ef 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -359,7 +359,6 @@ and { } return BisonParserImpl::token::KW_AND; } -"&&" { return BisonParserImpl::token::KW_AND; } anonymization { return BisonParserImpl::token::KW_ANONYMIZATION; } any { return BisonParserImpl::token::KW_ANY; } array { return BisonParserImpl::token::KW_ARRAY; } @@ -419,6 +418,7 @@ descriptor { return BisonParserImpl::token::KW_DESCRIPTOR; } describe { return BisonParserImpl::token::KW_DESCRIBE; } deterministic { return BisonParserImpl::token::KW_DETERMINISTIC; } distinct { return BisonParserImpl::token::KW_DISTINCT; } +div { return BisonParserImpl::token::KW_DIVIDE; } do { return BisonParserImpl::token::KW_DO; } drop { return BisonParserImpl::token::KW_DROP; } else { return BisonParserImpl::token::KW_ELSE; } @@ -872,13 +872,13 @@ zone { return BisonParserImpl::token::KW_ZONE; } } ">=" { return BisonParserImpl::token::KW_GREATER_EQUALS; } "||" { return BisonParserImpl::token::KW_CONCAT_OP; } +"&&" { return BisonParserImpl::token::KW_AND; } "|" { return '|'; } "^" { return '^'; } "&" { return '&'; } "+" { return '+'; } "-" { return '-'; } "/" { return '/'; } -div { return BisonParserImpl::token::KW_DIVIDE; } "%" { return '%'; } "~" { return '~'; } "?" { return '?'; } diff --git a/zetasql/parser/testdata/operator_precedence.test b/zetasql/parser/testdata/operator_precedence.test index ade1be4ae..f594023fe 100644 --- a/zetasql/parser/testdata/operator_precedence.test +++ b/zetasql/parser/testdata/operator_precedence.test @@ -108,6 +108,36 @@ SELECT (x + y) / z == +select x + y DIV z, (x + y) DIV z +-- +QueryStatement [0-33] + Query [0-33] + Select [0-33] + SelectList [7-33] + SelectColumn [7-18] + BinaryExpression(+) [7-18] + PathExpression [7-8] + Identifier(x) [7-8] + BinaryExpression(/) [11-18] + PathExpression [11-12] + Identifier(y) [11-12] + PathExpression [17-18] + Identifier(z) [17-18] + SelectColumn [20-33] + BinaryExpression(/) [20-33] + BinaryExpression(+) [21-26] + PathExpression [21-22] + Identifier(x) [21-22] + PathExpression [25-26] + Identifier(y) [25-26] + PathExpression [32-33] + Identifier(z) [32-33] +-- +SELECT + x + y / z, + (x + y) / z +== + select x / y + y * x / z -- QueryStatement [0-24] From 74ec9e528c39ff1b0af551fc8abc3069b9bd301d Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Wed, 2 Jun 2021 15:23:37 +0800 Subject: [PATCH 07/15] add '%' operator --- zetasql/parser/bison_parser.y | 1 + zetasql/parser/parse_tree.cc | 2 ++ zetasql/parser/parse_tree_manual.h | 1 + .../parser/testdata/operator_precedence.test | 30 +++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index 38611825a..00c1865b4 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -5228,6 +5228,7 @@ multiplicative_operator: "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } | "DIV" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } + | "%" { $$ = zetasql::ASTBinaryExpression::MOD; } ; // Returns ShiftOperator to indicate the operator type. diff --git a/zetasql/parser/parse_tree.cc b/zetasql/parser/parse_tree.cc index 4104bd142..116e4fef6 100644 --- a/zetasql/parser/parse_tree.cc +++ b/zetasql/parser/parse_tree.cc @@ -744,6 +744,8 @@ std::string ASTBinaryExpression::GetSQLForOperator() const { return "*"; case DIVIDE: return "/"; + case MOD: + return "%"; case CONCAT_OP: return "||"; case DISTINCT: diff --git a/zetasql/parser/parse_tree_manual.h b/zetasql/parser/parse_tree_manual.h index 037f9ca42..9768e58f2 100644 --- a/zetasql/parser/parse_tree_manual.h +++ b/zetasql/parser/parse_tree_manual.h @@ -1838,6 +1838,7 @@ class ASTBinaryExpression final : public ASTExpression { MINUS, // "-" MULTIPLY, // "*" DIVIDE, // "/" + MOD, // "%" CONCAT_OP, // "||" DISTINCT, // "IS DISTINCT FROM" }; diff --git a/zetasql/parser/testdata/operator_precedence.test b/zetasql/parser/testdata/operator_precedence.test index f594023fe..b472702ab 100644 --- a/zetasql/parser/testdata/operator_precedence.test +++ b/zetasql/parser/testdata/operator_precedence.test @@ -138,6 +138,36 @@ SELECT (x + y) / z == +select x + y % z, (x + y) % z +-- +QueryStatement [0-29] + Query [0-29] + Select [0-29] + SelectList [7-29] + SelectColumn [7-16] + BinaryExpression(+) [7-16] + PathExpression [7-8] + Identifier(x) [7-8] + BinaryExpression(%) [11-16] + PathExpression [11-12] + Identifier(y) [11-12] + PathExpression [15-16] + Identifier(z) [15-16] + SelectColumn [18-29] + BinaryExpression(%) [18-29] + BinaryExpression(+) [19-24] + PathExpression [19-20] + Identifier(x) [19-20] + PathExpression [23-24] + Identifier(y) [23-24] + PathExpression [28-29] + Identifier(z) [28-29] +-- +SELECT + x + y % z, + (x + y) % z +== + select x / y + y * x / z -- QueryStatement [0-24] From 49ac1d32ad48276b6a131300066e60b5b2e84285 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Wed, 2 Jun 2021 16:39:09 +0800 Subject: [PATCH 08/15] rm '&&' operator --- zetasql/parser/flex_tokenizer.l | 1 - 1 file changed, 1 deletion(-) diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index a854d44ef..2c3d965dc 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -872,7 +872,6 @@ zone { return BisonParserImpl::token::KW_ZONE; } } ">=" { return BisonParserImpl::token::KW_GREATER_EQUALS; } "||" { return BisonParserImpl::token::KW_CONCAT_OP; } -"&&" { return BisonParserImpl::token::KW_AND; } "|" { return '|'; } "^" { return '^'; } "&" { return '&'; } From b06b66938fbd89bd4eb3b393071e8a1567af181f Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Wed, 2 Jun 2021 17:53:42 +0800 Subject: [PATCH 09/15] add `mod` operator as synonym to `%` --- zetasql/parser/bison_parser.y | 9 +++- zetasql/parser/flex_tokenizer.l | 1 + zetasql/parser/keywords.cc | 1 + zetasql/parser/keywords_test.cc | 2 +- .../testdata/create_materialized_view.test | 4 +- zetasql/parser/testdata/create_table.test | 8 ++-- zetasql/parser/testdata/in.test | 46 +++++++++---------- 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index 00c1865b4..ddffdf791 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -561,7 +561,7 @@ class DashedIdentifierTmpNode final : public zetasql::ASTNode { %left "<<" ">>" %left "+" "-" %left "||" -%left "*" "/" "DIV" "%" +%left "*" "/" "DIV" "%" "MOD" %left UNARY_PRECEDENCE // For all unary operators %precedence DOUBLE_AT_PRECEDENCE // Needs to appear before "." %left PRIMARY_PRECEDENCE "(" ")" "[" "]" "." // For ., .(...), [], etc. @@ -712,6 +712,7 @@ using zetasql::ASTDropStatement; %token KW_LIMIT "LIMIT" %token KW_LOOKUP "LOOKUP" %token KW_MERGE "MERGE" +%token KW_MOD "MOD" %token KW_NATURAL "NATURAL" %token KW_NEW "NEW" %token KW_NO "NO" @@ -5229,6 +5230,7 @@ multiplicative_operator: | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } | "DIV" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } | "%" { $$ = zetasql::ASTBinaryExpression::MOD; } + | "MOD" { $$ = zetasql::ASTBinaryExpression::MOD; } ; // Returns ShiftOperator to indicate the operator type. @@ -6355,6 +6357,10 @@ function_name_from_keyword: { $$ = parser->MakeIdentifier(@1, parser->GetInputText(@1)); } + | "MOD" + { + $$ = parser->MakeIdentifier(@1, parser->GetInputText(@1)); + } ; // These rules have "expression" as their first part rather than @@ -7110,6 +7116,7 @@ reserved_keyword_rule: | "LIMIT" | "LOOKUP" | "MERGE" + | "MOD" | "NATURAL" | "NEW" | "NOT" diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index 2c3d965dc..7320edefa 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -515,6 +515,7 @@ maxsize { return BisonParserImpl::token::KW_MAXSIZE; } merge { return BisonParserImpl::token::KW_MERGE; } message { return BisonParserImpl::token::KW_MESSAGE; } min { return BisonParserImpl::token::KW_MIN; } +mod { return BisonParserImpl::token::KW_MOD; } model { return BisonParserImpl::token::KW_MODEL; } module { return BisonParserImpl::token::KW_MODULE; } natural { return BisonParserImpl::token::KW_NATURAL; } diff --git a/zetasql/parser/keywords.cc b/zetasql/parser/keywords.cc index 3556c9f3b..54f173443 100644 --- a/zetasql/parser/keywords.cc +++ b/zetasql/parser/keywords.cc @@ -191,6 +191,7 @@ constexpr KeywordInfoPOD kAllKeywords[] = { {"maxsize", KW_MAXSIZE}, {"message", KW_MESSAGE}, {"min", KW_MIN}, + {"mod", KW_MOD, KeywordInfo::kReserved}, {"model", KW_MODEL}, {"module", KW_MODULE}, {"merge", KW_MERGE, KeywordInfo::kReserved}, diff --git a/zetasql/parser/keywords_test.cc b/zetasql/parser/keywords_test.cc index a0554563c..a8c79f8c0 100644 --- a/zetasql/parser/keywords_test.cc +++ b/zetasql/parser/keywords_test.cc @@ -237,7 +237,7 @@ TEST(ParserTest, DontAddNewReservedKeywords) { // allows new queries to work that will not work on older code. // Before changing this, co-ordinate with all engines to make sure the change // is done safely. - EXPECT_EQ(100 /* CAUTION */, num_reserved); + EXPECT_EQ(101 /* CAUTION */, num_reserved); } } // namespace diff --git a/zetasql/parser/testdata/create_materialized_view.test b/zetasql/parser/testdata/create_materialized_view.test index cba0aa337..b37c53e89 100644 --- a/zetasql/parser/testdata/create_materialized_view.test +++ b/zetasql/parser/testdata/create_materialized_view.test @@ -151,7 +151,7 @@ CreateMaterializedViewStatement [0-109] ClusterBy [68-89] FunctionCall [79-89] PathExpression [79-82] - Identifier(mod) [79-82] + Identifier(`mod`) [79-82] PathExpression [83-85] Identifier(c1) [83-85] IntLiteral(2) [87-88] @@ -165,7 +165,7 @@ CreateMaterializedViewStatement [0-109] PathExpression [107-109] Identifier(t2) [107-109] -- -CREATE MATERIALIZED VIEW mv PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY mod(c1, 2) AS +CREATE MATERIALIZED VIEW mv PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY `mod`(c1, 2) AS SELECT * FROM diff --git a/zetasql/parser/testdata/create_table.test b/zetasql/parser/testdata/create_table.test index 1aa0dc396..53e94f1e5 100644 --- a/zetasql/parser/testdata/create_table.test +++ b/zetasql/parser/testdata/create_table.test @@ -2023,7 +2023,7 @@ CreateTableStatement [0-117] ClusterBy [96-117] FunctionCall [107-117] PathExpression [107-110] - Identifier(mod) [107-110] + Identifier(`mod`) [107-110] PathExpression [111-113] Identifier(c1) [111-113] IntLiteral(2) [115-116] @@ -2033,7 +2033,7 @@ CREATE TABLE t1 c1 int32, c2 int64, c3 string -) PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY mod(c1, 2) +) PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY `mod`(c1, 2) == @@ -2081,7 +2081,7 @@ CreateTableStatement [0-136] ClusterBy [96-117] FunctionCall [107-117] PathExpression [107-110] - Identifier(mod) [107-110] + Identifier(`mod`) [107-110] PathExpression [111-113] Identifier(c1) [111-113] IntLiteral(2) [115-116] @@ -2095,7 +2095,7 @@ CREATE TABLE t1 c1 int32, c2 int64, c3 string -) PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY mod(c1, 2) OPTIONS(foo = true) +) PARTITION BY c1, c2 + CAST(c3 AS int64) CLUSTER BY `mod`(c1, 2) OPTIONS(foo = true) == diff --git a/zetasql/parser/testdata/in.test b/zetasql/parser/testdata/in.test index 78b447549..d2c540cb4 100644 --- a/zetasql/parser/testdata/in.test +++ b/zetasql/parser/testdata/in.test @@ -159,34 +159,34 @@ SELECT col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) == -select col IN ('a') && col ! BETWEEN x + y AND f(x,y) --- -QueryStatement [0-53] - Query [0-53] - Select [0-53] - SelectList [7-53] - SelectColumn [7-53] - AndExpr [7-53] +select col IN ('a') AND col ! BETWEEN x + y AND f(x,y) +-- +QueryStatement [0-54] + Query [0-54] + Select [0-54] + SelectList [7-54] + SelectColumn [7-54] + AndExpr [7-54] InExpression(IN) [11-19] PathExpression [7-10] Identifier(col) [7-10] InList [15-18] StringLiteral('a') [15-18] - BetweenExpression(NOT BETWEEN) [29-53] - PathExpression [23-26] - Identifier(col) [23-26] - BinaryExpression(+) [37-42] - PathExpression [37-38] - Identifier(x) [37-38] - PathExpression [41-42] - Identifier(y) [41-42] - FunctionCall [47-53] - PathExpression [47-48] - Identifier(f) [47-48] - PathExpression [49-50] - Identifier(x) [49-50] - PathExpression [51-52] - Identifier(y) [51-52] + BetweenExpression(NOT BETWEEN) [30-54] + PathExpression [24-27] + Identifier(col) [24-27] + BinaryExpression(+) [38-43] + PathExpression [38-39] + Identifier(x) [38-39] + PathExpression [42-43] + Identifier(y) [42-43] + FunctionCall [48-54] + PathExpression [48-49] + Identifier(f) [48-49] + PathExpression [50-51] + Identifier(x) [50-51] + PathExpression [52-53] + Identifier(y) [52-53] -- SELECT col IN ('a') AND col NOT BETWEEN x + y AND f(x, y) From f00d90129018487afbe859d49d2303fd49ad6a7c Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Mon, 7 Jun 2021 15:27:23 +0800 Subject: [PATCH 10/15] feat: "div" is integer division --- zetasql/parser/bison_parser.y | 4 ++-- zetasql/parser/flex_tokenizer.l | 2 +- zetasql/parser/keywords.cc | 2 +- zetasql/parser/parse_tree.cc | 2 ++ zetasql/parser/parse_tree_manual.h | 1 + zetasql/parser/testdata/operator_precedence.test | 8 ++++---- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/zetasql/parser/bison_parser.y b/zetasql/parser/bison_parser.y index ddffdf791..ede18f474 100644 --- a/zetasql/parser/bison_parser.y +++ b/zetasql/parser/bison_parser.y @@ -672,7 +672,7 @@ using zetasql::ASTDropStatement; %token KW_DEFINE "DEFINE" %token KW_DESC "DESC" %token KW_DISTINCT "DISTINCT" -%token KW_DIVIDE "DIV" +%token KW_IDIVIDE "DIV" %token KW_ELSE "ELSE" %token KW_END "END" %token KW_ENUM "ENUM" @@ -5228,7 +5228,7 @@ additive_operator: multiplicative_operator: "*" { $$ = zetasql::ASTBinaryExpression::MULTIPLY; } | "/" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } - | "DIV" { $$ = zetasql::ASTBinaryExpression::DIVIDE; } + | "DIV" { $$ = zetasql::ASTBinaryExpression::IDIVIDE; } | "%" { $$ = zetasql::ASTBinaryExpression::MOD; } | "MOD" { $$ = zetasql::ASTBinaryExpression::MOD; } ; diff --git a/zetasql/parser/flex_tokenizer.l b/zetasql/parser/flex_tokenizer.l index 7320edefa..8231fc346 100644 --- a/zetasql/parser/flex_tokenizer.l +++ b/zetasql/parser/flex_tokenizer.l @@ -418,7 +418,7 @@ descriptor { return BisonParserImpl::token::KW_DESCRIPTOR; } describe { return BisonParserImpl::token::KW_DESCRIBE; } deterministic { return BisonParserImpl::token::KW_DETERMINISTIC; } distinct { return BisonParserImpl::token::KW_DISTINCT; } -div { return BisonParserImpl::token::KW_DIVIDE; } +div { return BisonParserImpl::token::KW_IDIVIDE; } do { return BisonParserImpl::token::KW_DO; } drop { return BisonParserImpl::token::KW_DROP; } else { return BisonParserImpl::token::KW_ELSE; } diff --git a/zetasql/parser/keywords.cc b/zetasql/parser/keywords.cc index 54f173443..a36ca1258 100644 --- a/zetasql/parser/keywords.cc +++ b/zetasql/parser/keywords.cc @@ -111,7 +111,7 @@ constexpr KeywordInfoPOD kAllKeywords[] = { {"descriptor", KW_DESCRIPTOR}, {"deterministic", KW_DETERMINISTIC}, {"distinct", KW_DISTINCT, KeywordInfo::kReserved}, - {"div", KW_DIVIDE, KeywordInfo::kReserved}, + {"div", KW_IDIVIDE, KeywordInfo::kReserved}, {"do", KW_DO}, {"drop", KW_DROP}, {"else", KW_ELSE, KeywordInfo::kReserved}, diff --git a/zetasql/parser/parse_tree.cc b/zetasql/parser/parse_tree.cc index 116e4fef6..6fda1b7da 100644 --- a/zetasql/parser/parse_tree.cc +++ b/zetasql/parser/parse_tree.cc @@ -744,6 +744,8 @@ std::string ASTBinaryExpression::GetSQLForOperator() const { return "*"; case DIVIDE: return "/"; + case IDIVIDE: + return "DIV"; case MOD: return "%"; case CONCAT_OP: diff --git a/zetasql/parser/parse_tree_manual.h b/zetasql/parser/parse_tree_manual.h index 9768e58f2..193586635 100644 --- a/zetasql/parser/parse_tree_manual.h +++ b/zetasql/parser/parse_tree_manual.h @@ -1838,6 +1838,7 @@ class ASTBinaryExpression final : public ASTExpression { MINUS, // "-" MULTIPLY, // "*" DIVIDE, // "/" + IDIVIDE, // "div", integer division MOD, // "%" CONCAT_OP, // "||" DISTINCT, // "IS DISTINCT FROM" diff --git a/zetasql/parser/testdata/operator_precedence.test b/zetasql/parser/testdata/operator_precedence.test index b472702ab..93ff2bba9 100644 --- a/zetasql/parser/testdata/operator_precedence.test +++ b/zetasql/parser/testdata/operator_precedence.test @@ -118,13 +118,13 @@ QueryStatement [0-33] BinaryExpression(+) [7-18] PathExpression [7-8] Identifier(x) [7-8] - BinaryExpression(/) [11-18] + BinaryExpression(DIV) [11-18] PathExpression [11-12] Identifier(y) [11-12] PathExpression [17-18] Identifier(z) [17-18] SelectColumn [20-33] - BinaryExpression(/) [20-33] + BinaryExpression(DIV) [20-33] BinaryExpression(+) [21-26] PathExpression [21-22] Identifier(x) [21-22] @@ -134,8 +134,8 @@ QueryStatement [0-33] Identifier(z) [32-33] -- SELECT - x + y / z, - (x + y) / z + x + y DIV z, + (x + y) DIV z == select x + y % z, (x + y) % z From c65a1c280f6b8c2a0656f701343cc12498a493af Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Mon, 7 Jun 2021 15:44:24 +0800 Subject: [PATCH 11/15] fix: reserved keywords number test --- zetasql/parser/keywords_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetasql/parser/keywords_test.cc b/zetasql/parser/keywords_test.cc index a8c79f8c0..3bb0235d8 100644 --- a/zetasql/parser/keywords_test.cc +++ b/zetasql/parser/keywords_test.cc @@ -237,7 +237,7 @@ TEST(ParserTest, DontAddNewReservedKeywords) { // allows new queries to work that will not work on older code. // Before changing this, co-ordinate with all engines to make sure the change // is done safely. - EXPECT_EQ(101 /* CAUTION */, num_reserved); + EXPECT_EQ(102 /* CAUTION */, num_reserved); } } // namespace From 6be4593a3f2310a9afafc9fefb5fa8958be165d7 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 8 Jun 2021 14:01:38 +0800 Subject: [PATCH 12/15] add build_zetasql_parser.sh --- .github/workflows/main.yml | 18 +++++++++--------- build_zetasql_parser.sh | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 build_zetasql_parser.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd22c3304..568963c83 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,7 +116,7 @@ jobs: source /opt/rh/rh-python38/enable bazel test ${{ env.build_argv }} --test_summary=detailed --test_output=errors ${{ env.target }} - + linux-build: runs-on: ubuntu-latest container: @@ -140,7 +140,7 @@ jobs: source /opt/rh/devtoolset-8/enable source /opt/rh/rh-python38/enable bazel build ${{ env.target }} ${{ env.build_argv }} - + - name: Test run: | bazel test ${{ env.build_argv }} --test_summary=detailed ${{ env.target }} @@ -148,18 +148,18 @@ jobs: - name: build zetasql parser dependencies if: ${{ github.event_name == 'push' }} run: | - bazel query 'deps(${{ env.target }})' |grep //zetasql|xargs bazel build ${{ env.build_argv }} + bazel query 'deps(${{ env.target }})' |grep //zetasql|xargs bazel build ${{ env.build_argv }} bazel build "@com_googleapis_googleapis//:all" ${{ env.build_argv }} bazel build "@com_google_file_based_test_driver//file_based_test_driver:all" ${{ env.build_argv }} bazel build "@com_googlesource_code_re2//:re2" ${{ env.build_argv }} - + - name: Determine Version if: ${{ github.event_name == 'push' }} run: | VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') VERSION=$(echo $VERSION | sed -e 's/^v//') echo "TAG=$VERSION" >> $GITHUB_ENV - + - name: pack libzetasql if: ${{ github.event_name == 'push' }} run: | @@ -211,14 +211,14 @@ jobs: - name: test run: | bazel test ${{ env.build_argv }} --test_summary=detailed ${{ env.target }} - + - name: Determine Version if: ${{ github.event_name == 'push' }} run: | VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') VERSION=$(echo $VERSION | sed -e 's/^v//') echo "TAG=$VERSION" >> $GITHUB_ENV - + - name: pack libzetasql if: ${{ github.event_name == 'push' }} run: | @@ -232,7 +232,7 @@ jobs: with: path: libzetasql-*.tar.gz name: release-artifacts - + release: runs-on: ubuntu-latest needs: ["linux-build", "macos-build"] @@ -252,4 +252,4 @@ jobs: libzetasql-*.tar.gz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - \ No newline at end of file + diff --git a/build_zetasql_parser.sh b/build_zetasql_parser.sh new file mode 100644 index 000000000..41ea53073 --- /dev/null +++ b/build_zetasql_parser.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# build zetasql parser into static library +# Copyright 2021 4Paradigm +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eE + +export BAZEL_LINKOPTS: '-static-libstdc++:-lm' +export BAZEL_LINKLIBS: '-l%:libstdc++.a' + +TARGET='//zetasql/parser/...' +BUILD_ARGV='--features=-supports_dynamic_linker' + +bazel build "$TARGET" "$BUILD_ARGV" +bazel test "$TARGET" "$BUILD_ARGV" + +# explicitly build dependencies into static library +bazel query "deps($TARGET)" | grep //zetasql | xargs bazel build "$BUILD_ARGV" +bazel build "@com_googleapis_googleapis//:all" "$BUILD_ARGV" +bazel build "@com_google_file_based_test_driver//file_based_test_driver:all" "$BUILD_ARGV" +bazel build "@com_googlesource_code_re2//:re2" "$BUILD_ARGV" + +unset BAZEL_LINKLIBS +unset BAZEL_LINKOPTS From 5aa346f5eb07aae8bde519caad45b8ab47b02f9a Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 8 Jun 2021 16:31:48 +0800 Subject: [PATCH 13/15] fix: build_zetasql_parser.sh --- build_zetasql_parser.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 build_zetasql_parser.sh diff --git a/build_zetasql_parser.sh b/build_zetasql_parser.sh old mode 100644 new mode 100755 index 41ea53073..b0c5c8aec --- a/build_zetasql_parser.sh +++ b/build_zetasql_parser.sh @@ -18,8 +18,8 @@ set -eE -export BAZEL_LINKOPTS: '-static-libstdc++:-lm' -export BAZEL_LINKLIBS: '-l%:libstdc++.a' +export BAZEL_LINKOPTS='-static-libstdc++:-lm' +export BAZEL_LINKLIBS='-l%:libstdc++.a' TARGET='//zetasql/parser/...' BUILD_ARGV='--features=-supports_dynamic_linker' From 59ea61b1a4a6ae6ed2442f28dc542f15dc6da3df Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 8 Jun 2021 17:41:48 +0800 Subject: [PATCH 14/15] create table option test, json style --- zetasql/parser/testdata/create_table.test | 87 +++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/zetasql/parser/testdata/create_table.test b/zetasql/parser/testdata/create_table.test index 53e94f1e5..2d2858d0c 100644 --- a/zetasql/parser/testdata/create_table.test +++ b/zetasql/parser/testdata/create_table.test @@ -22,6 +22,93 @@ CREATE TABLE t1 ) == +# create table with distribution options (json style) +create table t1 (a int32, b string) +options (distribution = json r''' +{ + "partitionnum": 3, + "replicanum": 3, + "distribution": [ + { + "leader": "leader1" + "followers": ['fo1'] + }, + { + "leader": "leader2" + "followers": [] + }, + { + "leader": "leader1" + "followers": ['fo1', 'fo2'] + } + ] +} +'''); +-- +CreateTableStatement [0-409] + PathExpression [13-15] + Identifier(t1) [13-15] + TableElementList [16-35] + ColumnDefinition [17-24] + Identifier(a) [17-18] + SimpleColumnSchema [19-24] + PathExpression [19-24] + Identifier(int32) [19-24] + ColumnDefinition [26-34] + Identifier(b) [26-27] + SimpleColumnSchema [28-34] + PathExpression [28-34] + Identifier(string) [28-34] + OptionsList [44-409] + OptionsEntry [45-408] + Identifier(distribution) [45-57] + JSONLiteral(r''' +{ + "partitionnum": 3, + "replicanum": 3, + "distribution": [ + { + "leader": "leader1" + "followers": ['fo1'] + }, + { + "leader": "leader2" + "followers": [] + }, + { + "leader": "leader1" + "followers": ['fo1', 'fo2'] + } + ] +} +-- +CREATE TABLE t1 +( + a int32, + b string +) OPTIONS(distribution = JSON r''' +{ + "partitionnum": 3, + "replicanum": 3, + "distribution": [ + { + "leader": "leader1" + "followers": ['fo1'] + }, + { + "leader": "leader2" + "followers": [] + }, + { + "leader": "leader1" + "followers": ['fo1', 'fo2'] + } + ] +} +''' +) +== + create table t1 (a int32 options(option_a=1), b int64 options(option_b=2)) options(table_option=3); -- From 450aa080b32454542e7c798d9e3449eb8ebec28e Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 8 Jun 2021 18:07:18 +0800 Subject: [PATCH 15/15] create table distribution option, struct style --- zetasql/parser/testdata/create_table.test | 52 ++++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/zetasql/parser/testdata/create_table.test b/zetasql/parser/testdata/create_table.test index 2d2858d0c..60b70e058 100644 --- a/zetasql/parser/testdata/create_table.test +++ b/zetasql/parser/testdata/create_table.test @@ -81,6 +81,7 @@ CreateTableStatement [0-409] } ] } +''') [60-408] -- CREATE TABLE t1 ( @@ -105,8 +106,55 @@ CREATE TABLE t1 } ] } -''' -) +''') +== + + +create table t1 (a int32, b string) options ( partitionnum = 3, replicanum = 3, distribution = [ ('le1', [ 'fo1' ]), ('le2', []), ('le3', ['fo1', 'fo2']) ] ); +-- +CreateTableStatement [0-157] + PathExpression [13-15] + Identifier(t1) [13-15] + TableElementList [16-35] + ColumnDefinition [17-24] + Identifier(a) [17-18] + SimpleColumnSchema [19-24] + PathExpression [19-24] + Identifier(int32) [19-24] + ColumnDefinition [26-34] + Identifier(b) [26-27] + SimpleColumnSchema [28-34] + PathExpression [28-34] + Identifier(string) [28-34] + OptionsList [44-157] + OptionsEntry [46-62] + Identifier(partitionnum) [46-58] + IntLiteral(3) [61-62] + OptionsEntry [64-78] + Identifier(replicanum) [64-74] + IntLiteral(3) [77-78] + OptionsEntry [80-155] + Identifier(distribution) [80-92] + ArrayConstructor [95-155] + StructConstructorWithParens [97-115] + StringLiteral('le1') [98-103] + ArrayConstructor [105-114] + StringLiteral('fo1') [107-112] + StructConstructorWithParens [117-128] + StringLiteral('le2') [118-123] + ArrayConstructor [125-127] + StructConstructorWithParens [130-153] + StringLiteral('le3') [131-136] + ArrayConstructor [138-152] + StringLiteral('fo1') [139-144] + StringLiteral('fo2') [146-151] +-- +CREATE TABLE t1 +( + a int32, + b string +) OPTIONS(partitionnum = 3, replicanum = 3, distribution = ARRAY[('le1', ARRAY['fo1']), ('le2', ARRAY[]), + ('le3', ARRAY['fo1', 'fo2'])]) == create table t1 (a int32 options(option_a=1), b int64 options(option_b=2))