From 860b197ad747a60418cd6d05bbdce03b8c659794 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 8 Nov 2019 12:08:00 +0000 Subject: [PATCH 1/8] Initial AAE Fold definitions PB definitions for use in AAE Fold --- .gitignore | 1 + src/riak.proto | 1 - src/riak_kv.proto | 115 ++++++++++++++++++++++++++++++++++++++- src/riak_pb_messages.csv | 10 ++++ 4 files changed, 125 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f581c6a7..bebc2d62 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ src/riak_pb_messages.erl dialyzer_unhandled_warnings dialyzer_warnings .rebar/* +.DS_Store # Java target/* diff --git a/src/riak.proto b/src/riak.proto index 6050c7c4..3107d374 100644 --- a/src/riak.proto +++ b/src/riak.proto @@ -48,7 +48,6 @@ message RpbPair { optional bytes value = 2; } - // Get bucket properties request message RpbGetBucketReq { required bytes bucket = 1; diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 827c4c1a..28c9e88c 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -346,4 +346,117 @@ message RpbRTEReq { optional bytes type = 13; // Bucket type, if not set we assume the 'default' type } -// RTE response - not defined, will return a RpbRTEResp on success or RpbErrorResp on failure \ No newline at end of file +// RTE response - not defined, will return a RpbRTEResp on success or RpbErrorResp on failure + +// AAE Fold requests +message RpbAaeFoldMergeRootNValReq{ + required uint32 n_val = 1; +} + +message RpbAaeFoldMergeBranchNValReq{ + required uint32 n_val = 1; + repeated uint32 id_filter = 2; +} + +message RpbAaeFoldFetchClocksNValReq{ + required uint32 n_val = 1; + repeated uint32 id_filter = 2; + +} + +message RpbAaeFoldMergeTreesRangeReq{ + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required RpbTreeSize tree_size = 20; + required bool segment_filter = 30; + repeated uint32 id_filter = 31; + optional RpbTreeSize filter_tree_size = 32; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; + required bool use_prehash = 50; + optional uint32 init_vector = 51; +} + +message RpbAaeFoldFetchClocksRangeReq{ + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool segment_filter = 30; + repeated uint32 id_filter = 31; + optional RpbTreeSize filter_tree_size = 32; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; +} + +message RpbAaeFoldFindKeysReq{ + enum FindType { + sibling_count = 0; + object_size = 1; + } + + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; + required FindType finder = 50; + required uint32 find_limit = 51; +} + +message RpbAaeFoldObjectStatsReq{ + optional bytes type = 1; + required bytes bucket = 2; + required bool key_range = 10; + optional bytes start_key = 11; + optional bytes end_key = 12; + required bool modified_range = 40; + optional uint32 last_mod_start = 41; + optional uint32 last_mod_end = 42; +} + +message RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; +} + +message RpbKeysClocks { + optional bytes type = 1; + required bytes bucket = 2; + required bytes key = 3; + required bytes value = 4; +} + + +message RpbKeyCount{ + required bytes tag = 1; + optional uint32 order = 2; + required uint32 count = 3; +} + +message RpbAaeFoldTreeResp{ + required RpbTreeSize size = 1; + optional bytes level_one = 2; + repeated RpbPair level_two = 3; +} + +message RpbAaeFoldKeyClockResp{ + repeated RpbKeysClocks keys_clocks = 1; +} + +message RpbAaeFoldKeyCountResp{ + repeated RpbKeyCount key_count = 1; +} \ No newline at end of file diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index e6ea186f..4cf84161 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -70,6 +70,16 @@ 104,TsTtbMsg,riak_ts 200,RpbRTEReq,riak_kv 201,RpbRTEResp,riak_kv +210,RpbAaeFoldMergeRootNValReq,riak_kv +211,RpbAaeFoldMergeBranchNValReq,riak_kv +212,RpbAaeFoldFetchClocksNValReq,riak_kv +213,RpbAaeFoldMergeTreesRangeReq,riak_kv +214,RpbAaeFoldFetchClocksRangeReq,riak_kv +215,RpbAaeFoldFindKeysReq,riak_kv +216,RpbAaeFoldObjectStatsReq,riak_kv +220,RpbAaeFoldTreeResp,riak_kv +221,RpbAaeFoldKeyClockResp,riak_kv +222,RpbAaeFoldKeyIntResp,riak_kv 253,RpbAuthReq,riak 254,RpbAuthResp,riak 255,RpbStartTls,riak From b5b24302b936b59e163bf201f57b41fe217e9d28 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 11 Nov 2019 20:15:16 +0000 Subject: [PATCH 2/8] Standardise on plural/singular --- src/riak_kv.proto | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 28c9e88c..f1a80e97 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -433,15 +433,14 @@ message RpbTreeSize{ xlarge = 5; } -message RpbKeysClocks { +message RpbKeysClock { optional bytes type = 1; required bytes bucket = 2; required bytes key = 3; required bytes value = 4; } - -message RpbKeyCount{ +message RpbKeysCount{ required bytes tag = 1; optional uint32 order = 2; required uint32 count = 3; @@ -454,9 +453,9 @@ message RpbAaeFoldTreeResp{ } message RpbAaeFoldKeyClockResp{ - repeated RpbKeysClocks keys_clocks = 1; + repeated RpbKeysClock keys_clock = 1; } message RpbAaeFoldKeyCountResp{ - repeated RpbKeyCount key_count = 1; + repeated RpbKeysCount keys_count = 1; } \ No newline at end of file From 588c2824d8baee48ed7e9c4bced562fce5a15298 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 12 Nov 2019 13:26:46 +0000 Subject: [PATCH 3/8] Add KeyCount Response No longer referred to as KeyInt --- src/riak_pb_messages.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index 4cf84161..18c0592d 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -79,7 +79,7 @@ 216,RpbAaeFoldObjectStatsReq,riak_kv 220,RpbAaeFoldTreeResp,riak_kv 221,RpbAaeFoldKeyClockResp,riak_kv -222,RpbAaeFoldKeyIntResp,riak_kv +222,RpbAaeFoldKeyCountResp,riak_kv 253,RpbAuthReq,riak 254,RpbAuthResp,riak 255,RpbStartTls,riak From 4d8e982cece5a2ba6293ec339511e11211787edd Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 12 Nov 2019 15:18:43 +0000 Subject: [PATCH 4/8] Allow For more generic response messages Using a required type tag to differentiate --- src/riak_kv.proto | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index f1a80e97..35c513e5 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -433,7 +433,7 @@ message RpbTreeSize{ xlarge = 5; } -message RpbKeysClock { +message RpbKeysValue { optional bytes type = 1; required bytes bucket = 2; required bytes key = 3; @@ -452,10 +452,12 @@ message RpbAaeFoldTreeResp{ repeated RpbPair level_two = 3; } -message RpbAaeFoldKeyClockResp{ - repeated RpbKeysClock keys_clock = 1; +message RpbAaeFoldKeyValueResp{ + required bytes type = 1; + repeated RpbKeysValue keys_value = 2; } message RpbAaeFoldKeyCountResp{ - repeated RpbKeysCount keys_count = 1; + required bytes type = 1; + repeated RpbKeysCount keys_count = 2; } \ No newline at end of file From 20e84dc07002e3f880d02f7c22252d2bc76bea14 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 12 Nov 2019 21:19:09 +0000 Subject: [PATCH 5/8] Enumberate tree size Also resolve confusion between type (bucket) and type (response) --- src/riak_kv.proto | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 35c513e5..61262419 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -365,6 +365,15 @@ message RpbAaeFoldFetchClocksNValReq{ } message RpbAaeFoldMergeTreesRangeReq{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + optional bytes type = 1; required bytes bucket = 2; required bool key_range = 10; @@ -382,6 +391,15 @@ message RpbAaeFoldMergeTreesRangeReq{ } message RpbAaeFoldFetchClocksRangeReq{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + optional bytes type = 1; required bytes bucket = 2; required bool key_range = 10; @@ -424,15 +442,6 @@ message RpbAaeFoldObjectStatsReq{ optional uint32 last_mod_end = 42; } -message RpbTreeSize{ - xxsmall = 0; - xsmall = 1; - small = 2; - medium = 3; - large = 4; - xlarge = 5; -} - message RpbKeysValue { optional bytes type = 1; required bytes bucket = 2; @@ -447,17 +456,26 @@ message RpbKeysCount{ } message RpbAaeFoldTreeResp{ + enum RpbTreeSize{ + xxsmall = 0; + xsmall = 1; + small = 2; + medium = 3; + large = 4; + xlarge = 5; + } + required RpbTreeSize size = 1; optional bytes level_one = 2; repeated RpbPair level_two = 3; } message RpbAaeFoldKeyValueResp{ - required bytes type = 1; + required bytes response_type = 1; repeated RpbKeysValue keys_value = 2; } message RpbAaeFoldKeyCountResp{ - required bytes type = 1; + required bytes response_type = 1; repeated RpbKeysCount keys_count = 2; } \ No newline at end of file From f870071f7419498651c55f13090565bbfac65ca6 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Nov 2019 09:28:40 +0000 Subject: [PATCH 6/8] level_two of tree is repeated bytes --- src/riak_kv.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_kv.proto b/src/riak_kv.proto index 61262419..eec1ebcd 100644 --- a/src/riak_kv.proto +++ b/src/riak_kv.proto @@ -467,7 +467,7 @@ message RpbAaeFoldTreeResp{ required RpbTreeSize size = 1; optional bytes level_one = 2; - repeated RpbPair level_two = 3; + repeated bytes level_two = 3; } message RpbAaeFoldKeyValueResp{ From 55e6122a056bd75512860c5a18cb086bdd32bec9 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Nov 2019 10:38:54 +0000 Subject: [PATCH 7/8] Update riak_pb_messages.csv --- src/riak_pb_messages.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_pb_messages.csv b/src/riak_pb_messages.csv index 18c0592d..c59d53f3 100644 --- a/src/riak_pb_messages.csv +++ b/src/riak_pb_messages.csv @@ -78,7 +78,7 @@ 215,RpbAaeFoldFindKeysReq,riak_kv 216,RpbAaeFoldObjectStatsReq,riak_kv 220,RpbAaeFoldTreeResp,riak_kv -221,RpbAaeFoldKeyClockResp,riak_kv +221,RpbAaeFoldKeyValueResp,riak_kv 222,RpbAaeFoldKeyCountResp,riak_kv 253,RpbAuthReq,riak 254,RpbAuthResp,riak From 05b8c0af5449c8bdbc43b3299832bc69af973976 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 14 Nov 2019 16:36:39 +0000 Subject: [PATCH 8/8] Simplify travis test Now re-enabled --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61b7be3e..1299c76b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: erlang -notifications: - webhooks: http://basho-engbot.herokuapp.com/travis?key=be744efd0a393f30d77d2e0076c8456be272e209 - email: eng@basho.com +script: + - make test otp_release: - - 18.3 - - 17.5 - R16B03 - - R15B03