From dbc2061442e4de35c368fb870ee7711586da5fb5 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 12 Nov 2024 17:04:43 +0000 Subject: [PATCH] Manifest Entry to belong to leveled_pmanifest There are two manifests - leveled_pmanifest and leveled_imanifest. Both have manifest_entry() type objects, but these types are different. To avoid confusion don't include the pmanifest manifest_entry() within the global include file - be specific that it belongs to the leveled_pmanifest module --- include/leveled.hrl | 7 -- src/leveled_pclerk.erl | 178 +++++++++++++++++++++++--------------- src/leveled_penciller.erl | 30 +++---- src/leveled_pmanifest.erl | 58 ++++++++++++- src/leveled_sst.erl | 13 ++- src/leveled_tree.erl | 15 ++-- 6 files changed, 188 insertions(+), 113 deletions(-) diff --git a/include/leveled.hrl b/include/leveled.hrl index 5638f93e..6295b4c5 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -88,13 +88,6 @@ is_basement = false :: boolean(), timestamp :: integer()}). --record(manifest_entry, - {start_key :: tuple(), - end_key :: tuple(), - owner :: pid(), - filename :: string() | undefined, - bloom = none :: leveled_ebloom:bloom() | none}). - -record(cdb_options, {max_size :: pos_integer() | undefined, max_count :: pos_integer() | undefined, diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index 0b0d157c..107bcf15 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -222,16 +222,22 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> leveled_pmanifest:merge_lookup( Manifest, SrcLevel + 1, - Src#manifest_entry.start_key, - Src#manifest_entry.end_key + leveled_pmanifest:entry_startkey(Src), + leveled_pmanifest:entry_endkey(Src) ), Candidates = length(SinkList), leveled_log:log(pc008, [SrcLevel, Candidates]), case Candidates of 0 -> NewLevel = SrcLevel + 1, - leveled_log:log(pc009, [Src#manifest_entry.filename, NewLevel]), - leveled_sst:sst_switchlevels(Src#manifest_entry.owner, NewLevel), + leveled_log:log( + pc009, + [leveled_pmanifest:entry_filename(Src), NewLevel] + ), + leveled_sst:sst_switchlevels( + leveled_pmanifest:entry_owner(Src), + NewLevel + ), Man0 = leveled_pmanifest:switch_manifest_entry( Manifest, @@ -251,7 +257,11 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> notify_deletions([], _Penciller) -> ok; notify_deletions([Head|Tail], Penciller) -> - ok = leveled_sst:sst_setfordelete(Head#manifest_entry.owner, Penciller), + ok = + leveled_sst:sst_setfordelete( + leveled_pmanifest:entry_owner(Head), + Penciller + ), notify_deletions(Tail, Penciller). @@ -261,9 +271,12 @@ notify_deletions([Head|Tail], Penciller) -> %% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1 perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN, OptsSST) -> - leveled_log:log(pc010, [Src#manifest_entry.filename, NewSQN]), + leveled_log:log(pc010, [leveled_pmanifest:entry_filename(Src), NewSQN]), SrcList = [{next, Src, all}], - MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), + MaxSQN = + leveled_sst:sst_getmaxsequencenumber( + leveled_pmanifest:entry_owner(Src) + ), SinkLevel = SrcLevel + 1, SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel), Additions = @@ -321,13 +334,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> {ok, Pid, Reply, Bloom} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, Entry = - #manifest_entry{ - start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName, - bloom=Bloom - }, + leveled_pmanifest:new_entry( + SmallestKey, HighestKey, Pid, FileName, Bloom), leveled_log:log_timer(pc015, [], TS1), do_merge( KL1Rem, KL2Rem, @@ -342,7 +350,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> list(leveled_pmanifest:manifest_entry())) -> leveled_pmanifest:manifest_entry(). grooming_scorer([ME | MEs]) -> - InitTombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + InitTombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), {HighestTC, BestME} = grooming_scorer(InitTombCount, ME, MEs), leveled_log:log(pc024, [HighestTC]), BestME. @@ -350,7 +359,8 @@ grooming_scorer([ME | MEs]) -> grooming_scorer(HighestTC, BestME, []) -> {HighestTC, BestME}; grooming_scorer(HighestTC, BestME, [ME | MEs]) -> - TombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + TombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), case TombCount > HighestTC of true -> grooming_scorer(TombCount, ME, MEs); @@ -448,9 +458,9 @@ grooming_score_test() -> true), DSK = {o, <<"B">>, <<"SK">>, null}, DEK = {o, <<"E">>, <<"EK">>, null}, - ME1 = #manifest_entry{owner=PidL3_1, start_key = DSK, end_key = DEK}, - ME1B = #manifest_entry{owner=PidL3_1B, start_key = DSK, end_key = DEK}, - ME2 = #manifest_entry{owner=PidL3_2, start_key = DSK, end_key = DEK}, + ME1 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1, "dummyL3_1", none), + ME1B = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1B, "dummyL3_1B", none), + ME2 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_2, "dummyL3_2", none), ?assertMatch(ME1, grooming_scorer([ME1, ME2])), ?assertMatch(ME1, grooming_scorer([ME2, ME1])), % prefer the file with the tombstone @@ -467,64 +477,94 @@ merge_file_test() -> ok = filelib:ensure_dir("test/test_area/ledger_files/"), KL1_L1 = lists:sort(generate_randomkeys(8000, 0, 1000)), {ok, PidL1_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L1.sst", - 1, - KL1_L1, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L1.sst", + 1, + KL1_L1, + 999999, + #sst_options{} + ), KL1_L2 = lists:sort(generate_randomkeys(8000, 0, 250)), {ok, PidL2_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L2.sst", - 2, - KL1_L2, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L2.sst", + 2, + KL1_L2, + 999999, + #sst_options{} + ), KL2_L2 = lists:sort(generate_randomkeys(8000, 250, 250)), {ok, PidL2_2, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL2_L2.sst", - 2, - KL2_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL2_L2.sst", + 2, + KL2_L2, + 999999, + #sst_options{press_method = lz4} + ), KL3_L2 = lists:sort(generate_randomkeys(8000, 500, 250)), {ok, PidL2_3, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL3_L2.sst", - 2, - KL3_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL3_L2.sst", + 2, + KL3_L2, + 999999, + #sst_options{press_method = lz4} + ), KL4_L2 = lists:sort(generate_randomkeys(8000, 750, 250)), {ok, PidL2_4, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL4_L2.sst", - 2, - KL4_L2, - 999999, - #sst_options{press_method = lz4}), - E1 = #manifest_entry{owner = PidL1_1, - filename = "./KL1_L1.sst", - end_key = lists:last(KL1_L1), - start_key = lists:nth(1, KL1_L1)}, - E2 = #manifest_entry{owner = PidL2_1, - filename = "./KL1_L2.sst", - end_key = lists:last(KL1_L2), - start_key = lists:nth(1, KL1_L2)}, - E3 = #manifest_entry{owner = PidL2_2, - filename = "./KL2_L2.sst", - end_key = lists:last(KL2_L2), - start_key = lists:nth(1, KL2_L2)}, - E4 = #manifest_entry{owner = PidL2_3, - filename = "./KL3_L2.sst", - end_key = lists:last(KL3_L2), - start_key = lists:nth(1, KL3_L2)}, - E5 = #manifest_entry{owner = PidL2_4, - filename = "./KL4_L2.sst", - end_key = lists:last(KL4_L2), - start_key = lists:nth(1, KL4_L2)}, + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL4_L2.sst", + 2, + KL4_L2, + 999999, + #sst_options{press_method = lz4} + ), + E1 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L1), + lists:last(KL1_L1), + PidL1_1, + "./KL1_L1.sst", + none + ), + E2 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L2), + lists:last(KL1_L2), + PidL2_1, + "./KL1_L2.sst", + none + ), + E3 = + leveled_pmanifest:new_entry( + lists:nth(1, KL2_L2), + lists:last(KL2_L2), + PidL2_2, + "./KL2_L2.sst", + none + ), + E4 = + leveled_pmanifest:new_entry( + lists:nth(1, KL3_L2), + lists:last(KL3_L2), + PidL2_3, + "./KL3_L2.sst", + none + ), + E5 = + leveled_pmanifest:new_entry( + lists:nth(1, KL4_L2), + lists:last(KL4_L2), + PidL2_4, + "./KL4_L2.sst", + none + ), Man0 = leveled_pmanifest:new_manifest(), Man1 = leveled_pmanifest:insert_manifest_entry(Man0, 1, 2, E2), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 67fe82f9..0a432ae3 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -1137,14 +1137,7 @@ handle_cast( State = #state{manifest = Man, levelzero_constructor = L0C, clerk = Clerk}) when ?IS_DEF(Man), ?IS_DEF(L0C), ?IS_DEF(Clerk) -> leveled_log:log(p0029, []), - ManEntry = - #manifest_entry{ - start_key=StartKey, - end_key=EndKey, - owner=L0C, - filename=FN, - bloom=Bloom - }, + ManEntry = leveled_pmanifest:new_entry(StartKey, EndKey, L0C, FN, Bloom), ManifestSQN = leveled_pmanifest:get_manifest_sqn(Man) + 1, UpdMan = leveled_pmanifest:insert_manifest_entry(Man, ManifestSQN, 0, ManEntry), @@ -1342,9 +1335,12 @@ sst_filename(ManSQN, Level, Count) -> %%% Internal functions %%%============================================================================ +-type update_forcedlogs_fun() :: fun((pid(), list(atom())) -> ok). +-type update_loglevel_fun() :: fun((pid(), atom()) -> ok). + -spec update_clerk - (pid()|undefined, fun((pid(), atom()) -> ok), atom()) -> ok; - (pid()|undefined, fun((pid(), list(atom())) -> ok), list(atom())) -> ok. + (pid()|undefined, update_loglevel_fun(), atom()) -> ok; + (pid()|undefined, update_forcedlogs_fun(), list(atom())) -> ok. update_clerk(undefined, _F, _T) -> ok; update_clerk(Clerk, F, T) when is_pid(Clerk) -> @@ -1402,12 +1398,8 @@ start_from_file( {ok, L0Pid, {L0StartKey, L0EndKey}, Bloom} = L0Open, L0SQN = leveled_sst:sst_getmaxsequencenumber(L0Pid), L0Entry = - #manifest_entry{ - start_key = L0StartKey, - end_key = L0EndKey, - filename = L0FN, - owner = L0Pid, - bloom = Bloom}, + leveled_pmanifest:new_entry( + L0StartKey, L0EndKey, L0Pid, L0FN, Bloom), Manifest2 = leveled_pmanifest:insert_manifest_entry( Manifest1, ManSQN + 1, 0, L0Entry), @@ -1448,13 +1440,13 @@ shutdown_manifest(Manifest, L0Constructor) -> EntryCloseFun = fun(ME) -> Owner = - case is_record(ME, manifest_entry) of + case leveled_pmanifest:is_entry(ME) of true -> - ME#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME); false -> case ME of {_SK, ME0} -> - ME0#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME0); ME -> ME end diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index 893c66d2..63d50389 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -7,7 +7,7 @@ %% each level. This is fine for short-lived volume tests, but as the deeper %% levels are used there will be an exponential penalty. %% -%% The originial intention was to swap out this implementation for a +%% The original intention was to swap out this implementation for a %% multi-version ETS table - but that became complex. So one of two changes %% are pending: %% - Use a single version ES cache for lower levels (and not allow snapshots to @@ -53,6 +53,17 @@ get_sstpids/1 ]). +-export( + [ + new_entry/5, + entry_startkey/1, + entry_endkey/1, + entry_filename/1, + entry_owner/1, + is_entry/1 + ] +). + -export([ filepath/2 ]). @@ -106,6 +117,16 @@ blooms = new_blooms() :: blooms() }). +-record(manifest_entry, + { + start_key :: leveled_codec:object_key(), + end_key :: leveled_codec:object_key(), + owner :: pid(), + filename :: string(), + bloom = none :: leveled_ebloom:bloom() | none + } +). + -type snapshot() :: {pid(), non_neg_integer(), pos_integer(), pos_integer()}. -type manifest() :: #manifest{}. @@ -757,9 +778,42 @@ get_sstpids(Manifest) -> lists:foldl(FoldFun, [], lists:seq(0, Manifest#manifest.basement)). %%%============================================================================ -%%% Internal Functions +%%% Manifest Entry %%%============================================================================ +-spec new_entry( + leveled_codec:object_key(), + leveled_codec:object_key(), + pid(), + string(), + leveled_ebloom:bloom()|none) -> manifest_entry(). +new_entry(StartKey, EndKey, Owner, FileName, Bloom) -> + #manifest_entry{ + start_key = StartKey, + end_key = EndKey, + owner = Owner, + filename = FileName, + bloom = Bloom + }. + +-spec is_entry(any()) -> boolean(). +is_entry(ME) -> is_record(ME, manifest_entry). + +-spec entry_startkey(manifest_entry()) -> leveled_codec:object_key(). +entry_startkey(ME) -> ME#manifest_entry.start_key. + +-spec entry_endkey(manifest_entry()) -> leveled_codec:object_key(). +entry_endkey(ME) -> ME#manifest_entry.end_key. + +-spec entry_owner(manifest_entry()) -> pid(). +entry_owner(ME) -> ME#manifest_entry.owner. + +-spec entry_filename(manifest_entry()) -> string(). +entry_filename(#manifest_entry{filename = FN}) when ?IS_DEF(FN)-> FN. + +%%%============================================================================ +%%% Internal Functions +%%%============================================================================ -spec get_manifest_entry( {tuple(), manifest_entry()}|manifest_entry()) -> manifest_entry(). diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index f033ab16..8aa21a9b 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -1044,21 +1044,18 @@ expand_list_by_pointer( OtherPointers ++ Remainder ); expand_list_by_pointer( - {next, #manifest_entry{owner = SSTPid}, StartKey, EndKey}, - Tail, - _Width, - _SegChecker, - LowLastMod) when is_pid(SSTPid) -> + {next, ME, StartKey, EndKey}, Tail, _Width, _SegChecker, LowLastMod + ) -> % The first pointer is a pointer to a file - expand_list_by_pointer will % in this case convert this into list of pointers within that SST file % i.e. of the form {pointer, SSTPid, Slot, StartKey, EndKey} % This can then be further expanded by calling again to % expand_list_by_pointer + SSTPid = leveled_pmanifest:entry_owner(ME), leveled_log:log(sst10, [SSTPid, is_process_alive(SSTPid)]), ExpPointer = sst_getfilteredrange(SSTPid, StartKey, EndKey, LowLastMod), ExpPointer ++ Tail. - -spec split_localpointers( pid(), list(expandable_pointer())) -> {list(slot_pointer()), list(expandable_pointer())}. @@ -3915,13 +3912,13 @@ merge_tester(NewFunS, NewFunM) -> ML1 = [{ next, - #manifest_entry{owner = P1, start_key = DSK, end_key = DEK}, + leveled_pmanifest:new_entry(DSK, DEK, P1, "P1", none), FK1 }], ML2 = [{ next, - #manifest_entry{owner = P2, start_key = DSK, end_key = DEK}, + leveled_pmanifest:new_entry(DSK, DEK, P2, "P2", none), FK2 }], NewR = diff --git a/src/leveled_tree.erl b/src/leveled_tree.erl index 747fa9be..15cfde26 100644 --- a/src/leveled_tree.erl +++ b/src/leveled_tree.erl @@ -955,14 +955,13 @@ search_range_idx_test() -> {o_rkv,"Bucket1","Key1",null}, "<0.320.0>","./16_1_6.sst", none}}]}, {1,{{o_rkv,"Bucket1","Key1",null},1,nil,nil}}}}, - StartKeyFun = - fun(ME) -> - ME#manifest_entry.start_key - end, - R = search_range({o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}, - Tree, - StartKeyFun), + R = + search_range( + {o_rkv, "Bucket", null, null}, + {o_rkv, "Bucket", null, null}, + Tree, + fun leveled_pmanifest:entry_startkey/1 + ), ?assertMatch(1, length(R)). -endif.