From aaeac7ba36bf7561b4496c31d9fd37b6a7cfa825 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Nov 2024 13:37:13 +0000 Subject: [PATCH] Mas d34 i453 eqwalizer (#454) * Add eqwalizer and clear for codec & sst The eqwalizer errors highlighted the need in several places for type clarification. Within tests there are some issue where a type is assumed, and so ignore has been used to handle this rather than write more complex code to be explicit about the assumption. The handling of arrays isn't great by eqwalizer - to be specific about the content of array causes issues when initialising an array. Perhaps a type (map maybe) where one can be more explicit about types might be a better option (even if there is a minimal performance impact). The use of a ?TOMB_COUNT defined option complicated the code much more with eqwalizer. So for now, there is no developer option to disable ?TOMB_COUNT. Test fixes required where strings have been used for buckets/keys not binaries. The leveled_sst statem needs a different state record for starting when compared to other modes. The state record has been divided up to reflect this, to make type management easier. The impact on performance needs to be tested. * Update ct tests to support binary keys/buckets only * Eqwalizer for leveled_cdb and leveled_tictac As array is used in leveled_tictac - there is the same issue as with leveled_sst * Remove redundant indirection of leveled_rand A legacy of pre-20 OTP * Morde modules eqwalized ebloom/log/util/monitor * Eqwalize further modules elp eqwalize leveled_codec; elp eqwalize leveled_sst; elp eqwalize leveled_cdb; elp eqwalize leveled_tictac; elp eqwalize leveled_log; elp eqwalize leveled_monitor; elp eqwalize leveled_head; elp eqwalize leveled_ebloom; elp eqwalize leveled_iclerk All concurrently OK * Refactor unit tests to use binary() no string() in key Previously string() was allowed just to avoid having to change all these tests. Go through the pain now, as part of eqwalizing. * Add fixes for penciller, inker Add a new ?IS_DEF macro to replace =/= undefined. Now more explicit about primary, object and query keys * Further fixes Need to clarify functions used by runner - where keys , query keys and object keys are used * Further eqwalisation * Eqwalize leveled_pmanifest Also make implementation independent of choice of dict - i.e. one can save a manifest using dict for blooms/pending_deletions and then open a manifest with code that uses a different type. Allow for slow dict to be replaced with map. Would not be backwards compatible though, without further thought - i.e. if you upgrade then downgrade. Redundant code created by leveled_sst refactoring removed. * Fix backwards compatibility issues * Manifest Entry to belong to leveled_pmanifest There are two manifests - leveled_pmanifest and leveled_imanifest. Both have manifest_entry() type objects, but these types are different. To avoid confusion don't include the pmanifest manifest_entry() within the global include file - be specific that it belongs to the leveled_pmanifest module * Ignore elp file - large binary * Update src/leveled_pmem.erl Remove unnecessary empty list from type definition Co-authored-by: Thomas Arts --------- Co-authored-by: Thomas Arts --- .gitignore | 1 + include/leveled.hrl | 21 +- rebar.config | 11 +- src/leveled_bookie.erl | 1374 +++++++++++++++----------- src/leveled_cdb.erl | 475 +++++---- src/leveled_codec.erl | 269 +++-- src/leveled_ebloom.erl | 20 +- src/leveled_head.erl | 68 +- src/leveled_iclerk.erl | 427 ++++---- src/leveled_imanifest.erl | 43 +- src/leveled_inker.erl | 335 ++++--- src/leveled_log.erl | 4 +- src/leveled_monitor.erl | 17 +- src/leveled_pclerk.erl | 242 +++-- src/leveled_penciller.erl | 1142 ++++++++++++++------- src/leveled_pmanifest.erl | 921 ++++++++++------- src/leveled_pmem.erl | 158 +-- src/leveled_rand.erl | 28 - src/leveled_runner.erl | 169 ++-- src/leveled_sst.erl | 1041 +++++++++++-------- src/leveled_tictac.erl | 51 +- src/leveled_tree.erl | 34 +- src/leveled_util.erl | 2 +- test/end_to_end/appdefined_SUITE.erl | 4 +- test/end_to_end/basic_SUITE.erl | 395 ++++---- test/end_to_end/iterator_SUITE.erl | 456 +++++---- test/end_to_end/perf_SUITE.erl | 28 +- test/end_to_end/riak_SUITE.erl | 42 +- test/end_to_end/testutil.erl | 154 +-- test/end_to_end/tictac_SUITE.erl | 182 ++-- 30 files changed, 4779 insertions(+), 3335 deletions(-) delete mode 100644 src/leveled_rand.erl diff --git a/.gitignore b/.gitignore index ae9a11c8..667578a3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ cover cover_* .eqc-info leveled_data/* +elp diff --git a/include/leveled.hrl b/include/leveled.hrl index 55b82816..6295b4c5 100644 --- a/include/leveled.hrl +++ b/include/leveled.hrl @@ -70,6 +70,15 @@ %% Inker key type used for tombstones %%%============================================================================ +%%%============================================================================ +%%% Helper Function +%%%============================================================================ + +-define(IS_DEF(Attribute), Attribute =/= undefined). + +-if(?OTP_RELEASE < 26). +-type dynamic() :: any(). +-endif. %%%============================================================================ %%% Shared records @@ -79,13 +88,6 @@ is_basement = false :: boolean(), timestamp :: integer()}). --record(manifest_entry, - {start_key :: tuple() | undefined, - end_key :: tuple() | undefined, - owner :: pid()|list(), - filename :: string() | undefined, - bloom = none :: leveled_ebloom:bloom() | none}). - -record(cdb_options, {max_size :: pos_integer() | undefined, max_count :: pos_integer() | undefined, @@ -129,14 +131,15 @@ singlefile_compactionperc :: float()|undefined, maxrunlength_compactionperc :: float()|undefined, score_onein = 1 :: pos_integer(), - snaptimeout_long :: pos_integer() | undefined, + snaptimeout_long = 60 :: pos_integer(), monitor = {no_monitor, 0} :: leveled_monitor:monitor()}). -record(penciller_options, {root_path :: string() | undefined, sst_options = #sst_options{} :: #sst_options{}, - max_inmemory_tablesize :: integer() | undefined, + max_inmemory_tablesize = ?MIN_PCL_CACHE_SIZE + :: pos_integer(), start_snapshot = false :: boolean(), snapshot_query, bookies_pid :: pid() | undefined, diff --git a/rebar.config b/rebar.config index b76e2c21..1953a964 100644 --- a/rebar.config +++ b/rebar.config @@ -12,6 +12,14 @@ {eunit_opts, [verbose]}. +{project_plugins, [ + {eqwalizer_rebar3, + {git_subdir, + "https://github.com/whatsapp/eqwalizer.git", + {branch, "main"}, + "eqwalizer_rebar3"}} +]}. + {profiles, [{eqc, [{deps, [meck, fqc]}, {erl_opts, [debug_info, {d, 'EQC'}]}, @@ -28,7 +36,8 @@ {deps, [ {lz4, ".*", {git, "https://github.com/nhs-riak/erlang-lz4", {branch, "nhse-develop-3.4"}}}, - {zstd, ".*", {git, "https://github.com/nhs-riak/zstd-erlang", {branch, "nhse-develop"}}} + {zstd, ".*", {git, "https://github.com/nhs-riak/zstd-erlang", {branch, "nhse-develop"}}}, + {eqwalizer_support, {git_subdir, "https://github.com/whatsapp/eqwalizer.git", {branch, "main"}, "eqwalizer_support"}} ]}. {ct_opts, [{dir, ["test/end_to_end"]}]}. diff --git a/src/leveled_bookie.erl b/src/leveled_bookie.erl index a0f8ba70..4aee189a 100644 --- a/src/leveled_bookie.erl +++ b/src/leveled_bookie.erl @@ -148,7 +148,7 @@ min_sqn = infinity :: integer()|infinity, max_sqn = 0 :: integer()}). --record(state, {inker :: pid() | undefined, +-record(state, {inker :: pid() | null, penciller :: pid() | undefined, cache_size :: pos_integer() | undefined, cache_multiple :: pos_integer() | undefined, @@ -359,18 +359,29 @@ ]. -type load_item() :: - {leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges()}. + { + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + leveled_codec:sqn(), + dynamic(), + leveled_codec:journal_keychanges(), + integer() + }. -type initial_loadfun() :: fun((leveled_codec:journal_key(), - any(), + dynamic(), non_neg_integer(), - {non_neg_integer(), non_neg_integer(), ledger_cache()}, + {non_neg_integer(), non_neg_integer(), list(load_item())}, fun((any()) -> {binary(), non_neg_integer()})) -> - {loop|stop, list(load_item())}). + {loop|stop, + { + non_neg_integer(), + non_neg_integer(), + list(load_item()) + } + } + ). -export_type([initial_loadfun/0, ledger_cache/0]). @@ -421,7 +432,9 @@ book_start(RootPath, LedgerCacheSize, JournalSize, SyncStrategy) -> %% comments on the open_options() type book_start(Opts) -> - gen_server:start_link(?MODULE, [set_defaults(Opts)], []). + {ok, Bookie} = + gen_server:start_link(?MODULE, [set_defaults(Opts)], []), + {ok, Bookie}. -spec book_plainstart(list(tuple())) -> {ok, pid()}. @@ -429,7 +442,9 @@ book_start(Opts) -> %% @doc %% Start used in tests to start without linking book_plainstart(Opts) -> - gen_server:start(?MODULE, [set_defaults(Opts)], []). + {ok, Bookie} = + gen_server:start(?MODULE, [set_defaults(Opts)], []), + {ok, Bookie}. -spec book_tempput(pid(), leveled_codec:key(), leveled_codec:key(), any(), @@ -448,8 +463,8 @@ book_plainstart(Opts) -> %% reload_strategy. If expired objects are to be compacted entirely, then the %% history of KeyChanges will be lost on reload. -book_tempput(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) - when is_integer(TTL) -> +book_tempput( + Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL) when is_integer(TTL) -> book_put(Pid, Bucket, Key, Object, IndexSpecs, Tag, TTL). %% @doc - Standard PUT @@ -613,7 +628,7 @@ book_sqn(Pid, Bucket, Key) -> book_sqn(Pid, Bucket, Key, Tag) -> gen_server:call(Pid, {head, Bucket, Key, Tag, true}, infinity). --spec book_returnfolder(pid(), tuple()) -> {async, fun(() -> term())}. +-spec book_returnfolder(pid(), tuple()) -> {async, fun(() -> dynamic())}. %% @doc Folds over store - deprecated %% The tuple() is a query, and book_returnfolder will return an {async, Folder} @@ -685,13 +700,12 @@ book_returnfolder(Pid, RunnerType) -> FoldAccT :: {FoldFun, Acc}, Range :: {IndexField, Start, End}, TermHandling :: {ReturnTerms, TermRegex}) -> - {async, Runner::fun(() -> term())} + {async, Runner::fun(() -> dynamic())} when Bucket::term(), Key :: term(), StartKey::term(), FoldFun::fun((Bucket, Key | {IndexVal, Key}, Acc) -> Acc), - Key::term(), - Acc::term(), + Acc::dynamic(), IndexField::term(), IndexVal::term(), Start::IndexVal, @@ -728,10 +742,9 @@ book_indexfold(Pid, Bucket, FoldAccT, Range, TermHandling) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Constraint :: first | all, Bucket :: term(), - Acc :: term(), Runner :: fun(() -> Acc). book_bucketlist(Pid, Tag, FoldAccT, Constraint) -> RunnerType= @@ -758,7 +771,7 @@ book_bucketlist(Pid, Tag, FoldAccT, Constraint) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Runner :: fun(() -> Acc). @@ -772,7 +785,7 @@ book_keylist(Pid, Tag, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Runner :: fun(() -> Acc). @@ -790,7 +803,7 @@ book_keylist(Pid, Tag, Bucket, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), KeyRange :: {StartKey, EndKey} | all, StartKey :: Key, @@ -809,7 +822,7 @@ book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), KeyRange :: {StartKey, EndKey} | all, StartKey :: Key, @@ -838,7 +851,7 @@ book_keylist(Pid, Tag, Bucket, KeyRange, FoldAccT, TermRegex) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -860,7 +873,7 @@ book_objectfold(Pid, Tag, FoldAccT, SnapPreFold) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -884,7 +897,7 @@ book_objectfold(Pid, Tag, FoldAccT, SnapPreFold, Order) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -940,7 +953,7 @@ book_objectfold(Pid, Tag, Bucket, Limiter, FoldAccT, SnapPreFold) -> Tag :: leveled_codec:tag(), FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -975,7 +988,7 @@ book_headfold(Pid, Tag, FoldAccT, JournalCheck, SnapPreFold, SegmentList) -> EndKey :: Key, FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -1008,7 +1021,7 @@ book_headfold(Pid, Tag, Limiter, FoldAccT, JournalCheck, SnapPreFold, SegmentLis EndKey :: Key, FoldAccT :: {FoldFun, Acc}, FoldFun :: fun((Bucket, Key, Value, Acc) -> Acc), - Acc :: term(), + Acc :: dynamic(), Bucket :: term(), Key :: term(), Value :: term(), @@ -1040,10 +1053,9 @@ book_headfold(Pid, Tag, all, FoldAccT, JournalCheck, SnapPreFold, SegmentList, LastModRange, MaxObjectCount}, book_returnfolder(Pid, RunnerType). --spec book_snapshot(pid(), - store|ledger, - tuple()|undefined, - boolean()|undefined) -> {ok, pid(), pid()|null}. +-spec book_snapshot( + pid(), store|ledger, tuple()|no_lookup|undefined, boolean()) + -> {ok, pid(), pid()|null}. %% @doc create a snapshot of the store %% @@ -1151,9 +1163,8 @@ book_headstatus(Pid) -> %%% gen_server callbacks %%%============================================================================ --spec init([open_options()]) -> {ok, book_state()}. +-spec init([open_options()]) -> {ok, book_state()}|{stop, atom()}. init([Opts]) -> - leveled_rand:seed(), case {proplists:get_value(snapshot_bookie, Opts), proplists:get_value(root_path, Opts)} of {undefined, undefined} -> @@ -1262,16 +1273,21 @@ init([Opts]) -> end. -handle_call({put, Bucket, Key, Object, IndexSpecs, Tag, TTL, DataSync}, - From, State) when State#state.head_only == false -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), +handle_call( + {put, Bucket, Key, Object, IndexSpecs, Tag, TTL, DataSync}, + From, + State) + when State#state.head_only == false , Tag =/= ?HEAD_TAG -> + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), SWLR = os:timestamp(), SW0 = leveled_monitor:maybe_time(State#state.monitor), - {ok, SQN, ObjSize} = leveled_inker:ink_put(State#state.inker, - LedgerKey, - Object, - {IndexSpecs, TTL}, - DataSync), + {ok, SQN, ObjSize} = + leveled_inker:ink_put( + State#state.inker, + LedgerKey, + Object, + {IndexSpecs, TTL}, + DataSync), {T0, SW1} = leveled_monitor:step_time(SW0), Changes = preparefor_ledgercache( @@ -1302,9 +1318,13 @@ handle_call({mput, ObjectSpecs, TTL}, From, State) {ok, SQN} = leveled_inker:ink_mput(State#state.inker, dummy, {ObjectSpecs, TTL}), Changes = - preparefor_ledgercache(?INKT_MPUT, ?DUMMY, - SQN, null, length(ObjectSpecs), - {ObjectSpecs, TTL}), + preparefor_ledgercache( + ?INKT_MPUT, + ?DUMMY, + SQN, null, + length(ObjectSpecs), + {ObjectSpecs, TTL} + ), Cache0 = addto_ledgercache(Changes, State#state.ledger_cache), case State#state.slow_offer of true -> @@ -1324,7 +1344,7 @@ handle_call({mput, ObjectSpecs, TTL}, From, State) end; handle_call({get, Bucket, Key, Tag}, _From, State) when State#state.head_only == false -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), SW0 = leveled_monitor:maybe_time(State#state.monitor), {H0, _CacheHit} = fetch_head(LedgerKey, @@ -1370,7 +1390,7 @@ handle_call({get, Bucket, Key, Tag}, _From, State) handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) when State#state.head_lookup == true -> SW0 = leveled_monitor:maybe_time(State#state.monitor), - LK = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LK = leveled_codec:to_objectkey(Bucket, Key, Tag), {Head, CacheHit} = fetch_head(LK, State#state.penciller, @@ -1412,7 +1432,7 @@ handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) case {LedgerMD, SQNOnly} of {not_found, _} -> not_found; - {_, false} -> + {LedgerMD, false} when LedgerMD =/= null -> {ok, leveled_head:build_head(Tag, LedgerMD)}; {_, true} -> {ok, SQN} @@ -1426,14 +1446,18 @@ handle_call({head, Bucket, Key, Tag, SQNOnly}, _From, State) UpdJrnalCheckFreq -> {reply, Reply, State#state{ink_checking = UpdJrnalCheckFreq}} end; -handle_call({snapshot, SnapType, Query, LongRunning}, _From, State) -> +handle_call( + {snapshot, SnapType, Query, LongRunning}, + _From, + State = #state{penciller = Pcl}) + when is_pid(Pcl) -> % Snapshot the store, specifying if the snapshot should be long running % (i.e. will the snapshot be queued or be required for an extended period % e.g. many minutes) {ok, PclSnap, InkSnap} = snapshot_store( State#state.ledger_cache, - State#state.penciller, + Pcl, State#state.inker, State#state.monitor, SnapType, @@ -1494,9 +1518,11 @@ handle_call(hot_backup, _From, State) when State#state.head_only == false -> bookies_pid = self()}, {ok, Snapshot} = leveled_inker:ink_snapstart(InkerOpts), {reply, {async, BackupFun(Snapshot)}, State}; -handle_call(close, _From, State) -> - leveled_inker:ink_close(State#state.inker), - leveled_penciller:pcl_close(State#state.penciller), +handle_call( + close, _From, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + leveled_inker:ink_close(Inker), + leveled_penciller:pcl_close(Pcl), leveled_monitor:monitor_close(element(1, State#state.monitor)), {stop, normal, ok, State}; handle_call(destroy, _From, State=#state{is_snapshot=Snp}) when Snp == false -> @@ -1515,11 +1541,11 @@ handle_call(Msg, _From, State) -> {reply, {unsupported_message, element(1, Msg)}, State}. -handle_cast({log_level, LogLevel}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_loglevel(PCL, LogLevel), - ok = leveled_inker:ink_loglevel(INK, LogLevel), +handle_cast( + {log_level, LogLevel}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_loglevel(Pcl, LogLevel), + ok = leveled_inker:ink_loglevel(Inker, LogLevel), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1528,11 +1554,11 @@ handle_cast({log_level, LogLevel}, State) -> end, ok = leveled_log:set_loglevel(LogLevel), {noreply, State}; -handle_cast({add_logs, ForcedLogs}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_addlogs(PCL, ForcedLogs), - ok = leveled_inker:ink_addlogs(INK, ForcedLogs), +handle_cast( + {add_logs, ForcedLogs}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_addlogs(Pcl, ForcedLogs), + ok = leveled_inker:ink_addlogs(Inker, ForcedLogs), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1541,11 +1567,11 @@ handle_cast({add_logs, ForcedLogs}, State) -> end, ok = leveled_log:add_forcedlogs(ForcedLogs), {noreply, State}; -handle_cast({remove_logs, ForcedLogs}, State) -> - PCL = State#state.penciller, - INK = State#state.inker, - ok = leveled_penciller:pcl_removelogs(PCL, ForcedLogs), - ok = leveled_inker:ink_removelogs(INK, ForcedLogs), +handle_cast( + {remove_logs, ForcedLogs}, State = #state{inker = Inker, penciller = Pcl}) + when is_pid(Inker), is_pid(Pcl) -> + ok = leveled_penciller:pcl_removelogs(Pcl, ForcedLogs), + ok = leveled_inker:ink_removelogs(Inker, ForcedLogs), case element(1, State#state.monitor) of no_monitor -> ok; @@ -1661,8 +1687,8 @@ loadqueue_ledgercache(Cache) -> null|pid(), leveled_monitor:monitor(), store|ledger, - undefined|tuple(), - undefined|boolean()) -> + undefined|no_lookup|tuple(), + boolean()) -> {ok, pid(), pid()|null}. %% @doc %% Allow all a snapshot to be created from part of the store, preferably @@ -1679,7 +1705,7 @@ loadqueue_ledgercache(Cache) -> %% lookup is required but the range isn't defined then 'undefined' should be %% passed as the query snapshot_store( - LedgerCache, Penciller, Inker, Monitor, SnapType, Query, LongRunning) -> + LedgerCache, Penciller, Ink, Monitor, SnapType, Query, LongRunning) -> SW0 = leveled_monitor:maybe_time(Monitor), LedgerCacheReady = readycache_forsnapshot(LedgerCache, Query), BookiesMem = {LedgerCacheReady#ledger_cache.loader, @@ -1687,21 +1713,25 @@ snapshot_store( LedgerCacheReady#ledger_cache.min_sqn, LedgerCacheReady#ledger_cache.max_sqn}, PCLopts = - #penciller_options{start_snapshot = true, - source_penciller = Penciller, - snapshot_query = Query, - snapshot_longrunning = LongRunning, - bookies_pid = self(), - bookies_mem = BookiesMem}, + #penciller_options{ + start_snapshot = true, + source_penciller = Penciller, + snapshot_query = Query, + snapshot_longrunning = LongRunning, + bookies_pid = self(), + bookies_mem = BookiesMem + }, {TS0, SW1} = leveled_monitor:step_time(SW0), {ok, LedgerSnapshot} = leveled_penciller:pcl_snapstart(PCLopts), {TS1, _SW2} = leveled_monitor:step_time(SW1), ok = maybelog_snap_timing(Monitor, TS0, TS1), case SnapType of - store -> - InkerOpts = #inker_options{start_snapshot = true, - bookies_pid = self(), - source_inker = Inker}, + store when is_pid(Ink) -> + InkerOpts = + #inker_options{ + start_snapshot = true, + bookies_pid = self(), + source_inker = Ink}, {ok, JournalSnapshot} = leveled_inker:ink_snapstart(InkerOpts), {ok, LedgerSnapshot, JournalSnapshot}; ledger -> @@ -1797,8 +1827,13 @@ set_options(Opts, Monitor) -> ReloadStrategy = leveled_codec:inker_reload_strategy(AltStrategy), PCLL0CacheSize = - max(?MIN_PCL_CACHE_SIZE, - proplists:get_value(max_pencillercachesize, Opts)), + case proplists:get_value(max_pencillercachesize, Opts) of + P0CS when is_integer(P0CS), P0CS > ?MIN_PCL_CACHE_SIZE -> + P0CS; + _ -> + ?MIN_PCL_CACHE_SIZE + end, + RootPath = proplists:get_value(root_path, Opts), JournalFP = filename:join(RootPath, ?JOURNAL_FP), @@ -1896,7 +1931,13 @@ set_options(Opts, Monitor) -> %% previous snapshot should be used instead. Also the snapshot should not be %% closed as part of the post-query activity as the snapshot may be reused, and %% should be manually closed. -return_snapfun(State, SnapType, Query, LongRunning, SnapPreFold) -> +return_snapfun( + State = #state{penciller = Pcl, inker = Ink}, + SnapType, + Query, + LongRunning, + SnapPreFold) + when is_pid(Pcl), is_pid(Ink) -> CloseFun = fun(LS0, JS0) -> fun() -> @@ -1965,9 +2006,9 @@ get_runner(State, {index_query, Constraint, FoldAccT, Range, TermHandling}) -> {B, null} end, StartKey = - leveled_codec:to_ledgerkey(Bucket, ObjKey0, ?IDX_TAG, IdxFld, StartT), + leveled_codec:to_querykey(Bucket, ObjKey0, ?IDX_TAG, IdxFld, StartT), EndKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, IdxFld, EndT), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, IdxFld, EndT), SnapFun = return_snapfun(State, ledger, {StartKey, EndKey}, false, false), leveled_runner:index_query(SnapFun, {StartKey, EndKey, TermHandling}, @@ -2119,8 +2160,15 @@ get_deprecatedrunner(State, PartitionFilter). --spec return_ledger_keyrange(atom(), any(), tuple()|all) -> - {tuple(), tuple(), tuple()|no_lookup}. +-spec return_ledger_keyrange( + atom(), leveled_codec:key(), tuple()|all) + -> + { + leveled_codec:query_key(), + leveled_codec:query_key(), + {leveled_codec:query_key(), + leveled_codec:query_key()}|no_lookup + }. %% @doc %% Convert a range of binary keys into a ledger key range, returning %% {StartLK, EndLK, Query} where Query is to indicate whether the query @@ -2129,16 +2177,16 @@ return_ledger_keyrange(Tag, Bucket, KeyRange) -> {StartKey, EndKey, Snap} = case KeyRange of all -> - {leveled_codec:to_ledgerkey(Bucket, null, Tag), - leveled_codec:to_ledgerkey(Bucket, null, Tag), + {leveled_codec:to_querykey(Bucket, null, Tag), + leveled_codec:to_querykey(Bucket, null, Tag), false}; {StartTerm, <<"$all">>} -> - {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), - leveled_codec:to_ledgerkey(Bucket, null, Tag), + {leveled_codec:to_querykey(Bucket, StartTerm, Tag), + leveled_codec:to_querykey(Bucket, null, Tag), false}; {StartTerm, EndTerm} -> - {leveled_codec:to_ledgerkey(Bucket, StartTerm, Tag), - leveled_codec:to_ledgerkey(Bucket, EndTerm, Tag), + {leveled_codec:to_querykey(Bucket, StartTerm, Tag), + leveled_codec:to_querykey(Bucket, EndTerm, Tag), true} end, SnapQuery = @@ -2164,8 +2212,8 @@ maybe_longrunning(SW, Aspect) -> ok end. --spec readycache_forsnapshot(ledger_cache(), tuple()|no_lookup|undefined) - -> ledger_cache(). +-spec readycache_forsnapshot( + ledger_cache(), tuple()|no_lookup|undefined) -> ledger_cache(). %% @doc %% Strip the ledger cach back to only the relevant information needed in %% the query, and to make the cache a snapshot (and so not subject to changes @@ -2303,7 +2351,7 @@ journal_notfound(CheckFrequency, Inker, LK, SQN) -> {boolean(), integer()}. %% @doc Use a function to check if an item is found check_notfound(CheckFrequency, CheckFun) -> - case leveled_rand:uniform(?MAX_KEYCHECK_FREQUENCY) of + case rand:uniform(?MAX_KEYCHECK_FREQUENCY) of X when X =< CheckFrequency -> case CheckFun() of probably -> @@ -2316,28 +2364,32 @@ check_notfound(CheckFrequency, CheckFun) -> end. --spec preparefor_ledgercache(leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges()) - -> {leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}. +-spec preparefor_ledgercache( + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + non_neg_integer(), + any(), + integer(), + leveled_codec:journal_keychanges()) + -> {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}. %% @doc %% Prepare an object and its related key changes for addition to the Ledger %% via the Ledger Cache. -preparefor_ledgercache(?INKT_MPUT, - ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}) -> +preparefor_ledgercache(?INKT_MPUT, ?DUMMY, SQN, _O, _S, {ObjSpecs, TTL}) -> ObjChanges = leveled_codec:obj_objectspecs(ObjSpecs, SQN, TTL), {no_lookup, SQN, ObjChanges}; -preparefor_ledgercache(?INKT_KEYD, - LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL}) -> +preparefor_ledgercache( + ?INKT_KEYD, LedgerKey, SQN, _Obj, _Size, {IdxSpecs, TTL}) + when LedgerKey =/= ?DUMMY -> {Bucket, Key} = leveled_codec:from_ledgerkey(LedgerKey), KeyChanges = leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL), {no_lookup, SQN, KeyChanges}; -preparefor_ledgercache(_InkTag, - LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL}) -> +preparefor_ledgercache( + _InkTag, LedgerKey, SQN, Obj, Size, {IdxSpecs, TTL}) + when LedgerKey =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = leveled_codec:generate_ledgerkv(LedgerKey, SQN, Obj, Size, TTL), KeyChanges = @@ -2346,31 +2398,31 @@ preparefor_ledgercache(_InkTag, {KeyH, SQN, KeyChanges}. --spec recalcfor_ledgercache(leveled_codec:journal_key_tag()|null, - leveled_codec:ledger_key()|?DUMMY, - non_neg_integer(), any(), integer(), - leveled_codec:journal_keychanges(), - ledger_cache(), - pid()) - -> {leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}. +-spec recalcfor_ledgercache( + leveled_codec:journal_key_tag()|null, + leveled_codec:primary_key()|?DUMMY, + non_neg_integer(), + binary()|term(), + integer(), + leveled_codec:journal_keychanges(), + ledger_cache(), + pid()) + -> {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}. %% @doc %% When loading from the journal to the ledger, may hit a key which has the %% `recalc` strategy. Such a key needs to recalculate the key changes by %% comparison with the current state of the ledger, assuming it is a full %% journal entry (i.e. KeyDeltas which may be a result of previously running %% with a retain strategy should be ignored). -recalcfor_ledgercache(InkTag, - _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, - _LedgerCache, - _Penciller) - when InkTag == ?INKT_MPUT; InkTag == ?INKT_KEYD -> +recalcfor_ledgercache( + InkTag, _LedgerKey, SQN, _Obj, _Size, {_IdxSpecs, _TTL}, _LC, _Pcl) + when InkTag == ?INKT_MPUT; InkTag == ?INKT_KEYD -> {no_lookup, SQN, []}; -recalcfor_ledgercache(_InkTag, - LK, SQN, Obj, Size, {_IgnoreJournalIdxSpecs, TTL}, - LedgerCache, - Penciller) -> +recalcfor_ledgercache( + _InkTag, LK, SQN, Obj, Size, {_Ignore, TTL}, LedgerCache, Penciller) + when LK =/= ?DUMMY -> {Bucket, Key, MetaValue, {KeyH, _ObjH}, _LastMods} = leveled_codec:generate_ledgerkv(LK, SQN, Obj, Size, TTL), OldObject = @@ -2385,9 +2437,16 @@ recalcfor_ledgercache(_InkTag, not_present -> not_present; {LK, LV} -> - leveled_codec:get_metadata(LV) + case leveled_codec:get_metadata(LV) of + MDO when MDO =/= null -> + MDO + end + end, + UpdMetadata = + case leveled_codec:get_metadata(MetaValue) of + MDU when MDU =/= null -> + MDU end, - UpdMetadata = leveled_codec:get_metadata(MetaValue), IdxSpecs = leveled_head:diff_indexspecs(element(1, LK), UpdMetadata, OldMetadata), {KeyH, @@ -2396,11 +2455,12 @@ recalcfor_ledgercache(_InkTag, ++ leveled_codec:idx_indexspecs(IdxSpecs, Bucket, Key, SQN, TTL)}. --spec addto_ledgercache({leveled_codec:segment_hash(), - non_neg_integer(), - list(leveled_codec:ledger_kv())}, - ledger_cache()) - -> ledger_cache(). +-spec addto_ledgercache( + {leveled_codec:segment_hash(), + non_neg_integer(), + list(leveled_codec:ledger_kv())}, + ledger_cache()) + -> ledger_cache(). %% @doc %% Add a set of changes associated with a single sequence number (journal %% update) and key to the ledger cache. If the changes are not to be looked @@ -2412,12 +2472,13 @@ addto_ledgercache({H, SQN, KeyChanges}, Cache) -> min_sqn=min(SQN, Cache#ledger_cache.min_sqn), max_sqn=max(SQN, Cache#ledger_cache.max_sqn)}. --spec addto_ledgercache({integer()|no_lookup, - integer(), - list(leveled_codec:ledger_kv())}, - ledger_cache(), - loader) - -> ledger_cache(). +-spec addto_ledgercache( + {leveled_codec:segment_hash()|no_lookup, + integer(), + list(leveled_codec:ledger_kv())}, + ledger_cache(), + loader) + -> ledger_cache(). %% @doc %% Add a set of changes associated with a single sequence number (journal %% update) to the ledger cache. This is used explicitly when loading the @@ -2469,10 +2530,13 @@ maybepush_ledgercache(MaxCacheSize, MaxCacheMult, Cache, Penciller) -> TimeToPush = maybe_withjitter(CacheSize, MaxCacheSize, MaxCacheMult), if TimeToPush -> - CacheToLoad = {Tab, - Cache#ledger_cache.index, - Cache#ledger_cache.min_sqn, - Cache#ledger_cache.max_sqn}, + CacheToLoad = + { + Tab, + Cache#ledger_cache.index, + Cache#ledger_cache.min_sqn, + Cache#ledger_cache.max_sqn + }, case leveled_penciller:pcl_pushmem(Penciller, CacheToLoad) of ok -> Cache0 = #ledger_cache{}, @@ -2493,7 +2557,7 @@ maybepush_ledgercache(MaxCacheSize, MaxCacheMult, Cache, Penciller) -> %% a push should be maybe_withjitter( CacheSize, MaxCacheSize, MaxCacheMult) when CacheSize > MaxCacheSize -> - R = leveled_rand:uniform(MaxCacheMult * MaxCacheSize), + R = rand:uniform(MaxCacheMult * MaxCacheSize), (CacheSize - MaxCacheSize) > R; maybe_withjitter(_CacheSize, _MaxCacheSize, _MaxCacheMult) -> false. @@ -2515,7 +2579,8 @@ get_loadfun() -> _ -> {VBin, ValSize} = ExtractFun(ValueInJournal), % VBin may already be a term - {Obj, IdxSpecs} = leveled_codec:split_inkvalue(VBin), + {Obj, IdxSpecs} = + leveled_codec:revert_value_from_journal(VBin), case SQN of MaxSQN -> {stop, @@ -2546,11 +2611,14 @@ delete_path(DirPath) -> leveled_monitor:timing(), leveled_monitor:timing(), pos_integer()) -> ok. -maybelog_put_timing(_Monitor, no_timing, no_timing, no_timing, _Size) -> - ok; -maybelog_put_timing({Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) -> +maybelog_put_timing( + {Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) + when is_pid(Pid), + is_integer(InkTime), is_integer(PrepTime), is_integer(MemTime) -> leveled_monitor:add_stat( - Pid, {bookie_put_update, InkTime, PrepTime, MemTime, Size}). + Pid, {bookie_put_update, InkTime, PrepTime, MemTime, Size}); +maybelog_put_timing(_Monitor, _, _, _, _Size) -> + ok. -spec maybelog_head_timing( leveled_monitor:monitor(), @@ -2558,37 +2626,42 @@ maybelog_put_timing({Pid, _StatsFreq}, InkTime, PrepTime, MemTime, Size) -> leveled_monitor:timing(), boolean(), boolean()) -> ok. -maybelog_head_timing(_Monitor, no_timing, no_timing, _NF, _CH) -> - ok; -maybelog_head_timing({Pid, _StatsFreq}, FetchTime, _, true, _CH) -> - leveled_monitor:add_stat( - Pid, {bookie_head_update, FetchTime, not_found, 0}); -maybelog_head_timing({Pid, _StatsFreq}, FetchTime, RspTime, _NF, CH) -> +maybelog_head_timing({Pid, _StatsFreq}, FetchTime, RspTime, false, CH) + when is_pid(Pid), is_integer(FetchTime), is_integer(RspTime) -> CH0 = case CH of true -> 1; false -> 0 end, leveled_monitor:add_stat( - Pid, {bookie_head_update, FetchTime, RspTime, CH0}). + Pid, {bookie_head_update, FetchTime, RspTime, CH0}); +maybelog_head_timing({Pid, _StatsFreq}, FetchTime, _, true, _CH) + when is_pid(Pid), is_integer(FetchTime) -> + leveled_monitor:add_stat( + Pid, {bookie_head_update, FetchTime, not_found, 0}); +maybelog_head_timing(_Monitor, _, _, _NF, _CH) -> + ok. -spec maybelog_get_timing( leveled_monitor:monitor(), leveled_monitor:timing(), leveled_monitor:timing(), boolean()) -> ok. -maybelog_get_timing(_Monitor, no_timing, no_timing, _NF) -> - ok; -maybelog_get_timing({Pid, _StatsFreq}, HeadTime, _BodyTime, true) -> +maybelog_get_timing({Pid, _StatsFreq}, HeadTime, BodyTime, false) + when is_pid(Pid), is_integer(HeadTime), is_integer(BodyTime) -> + leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, BodyTime}); +maybelog_get_timing({Pid, _StatsFreq}, HeadTime, _BodyTime, true) + when is_pid(Pid), is_integer(HeadTime) -> leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, not_found}); -maybelog_get_timing({Pid, _StatsFreq}, HeadTime, BodyTime, false) -> - leveled_monitor:add_stat(Pid, {bookie_get_update, HeadTime, BodyTime}). - +maybelog_get_timing(_Monitor, _, _, _NF) -> + ok. -spec maybelog_snap_timing( leveled_monitor:monitor(), leveled_monitor:timing(), leveled_monitor:timing()) -> ok. -maybelog_snap_timing(_Monitor, no_timing, no_timing) -> - ok; -maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) -> - leveled_monitor:add_stat(Pid, {bookie_snap_update, BookieTime, PCLTime}). +maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) + when is_pid(Pid), is_integer(BookieTime), is_integer(PCLTime) -> + leveled_monitor:add_stat(Pid, {bookie_snap_update, BookieTime, PCLTime}); +maybelog_snap_timing(_Monitor, _, _) -> + ok. + %%%============================================================================ %%% Test @@ -2603,27 +2676,32 @@ maybelog_snap_timing({Pid, _StatsFreq}, BookieTime, PCLTime) -> book_returnactors(Pid) -> gen_server:call(Pid, return_actors). - reset_filestructure() -> RootPath = "test/test_area", leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), leveled_penciller:clean_testdir(RootPath ++ "/" ++ ?LEDGER_FP), RootPath. - generate_multiple_objects(Count, KeyNumber) -> generate_multiple_objects(Count, KeyNumber, []). generate_multiple_objects(0, _KeyNumber, ObjL) -> ObjL; generate_multiple_objects(Count, KeyNumber, ObjL) -> - Key = "Key" ++ integer_to_list(KeyNumber), - Value = leveled_rand:rand_bytes(256), - IndexSpec = [{add, "idx1_bin", "f" ++ integer_to_list(KeyNumber rem 10)}], - generate_multiple_objects(Count - 1, - KeyNumber + 1, - ObjL ++ [{Key, Value, IndexSpec}]). - + Key = list_to_binary("Key" ++ integer_to_list(KeyNumber)), + Value = crypto:strong_rand_bytes(256), + IndexSpec = + [ + { + add, <<"idx1_bin">>, + list_to_binary("f" ++ integer_to_list(KeyNumber rem 10)) + } + ], + generate_multiple_objects( + Count - 1, + KeyNumber + 1, + ObjL ++ [{Key, Value, IndexSpec}] + ). shutdown_test_() -> {timeout, 10, fun shutdown_tester/0}. @@ -2648,7 +2726,9 @@ shutdown_tester() -> timer:sleep(2000), ok = leveled_penciller:pcl_close(SnpPCL1), - ok = leveled_inker:ink_close(SnpJrnl1), + case SnpJrnl1 of + P when is_pid(P) -> ok = leveled_inker:ink_close(SnpJrnl1) + end, SW = os:timestamp(), receive ok -> ok end, WaitForShutDown = timer:now_diff(SW, os:timestamp()) div 1000, @@ -2662,83 +2742,105 @@ ttl_test() -> ObjL1 = generate_multiple_objects(100, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - lists:foreach(fun({K, V, _S}) -> - {ok, V} = book_get(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL1), - lists:foreach(fun({K, _V, _S}) -> - {ok, _} = book_head(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL1), + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + lists:foreach( + fun({K, V, _S}) -> + {ok, V} = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL1 + ), + lists:foreach( + fun({K, _V, _S}) -> + {ok, _} = book_head(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL1 + ), ObjL2 = generate_multiple_objects(100, 101), Past = leveled_util:integer_now() - 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Past) end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_get(Bookie1, "Bucket", K, ?STD_TAG) - end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_head(Bookie1, "Bucket", K, ?STD_TAG) - end, + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Past) + end, + ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_head(Bookie1, <<"Bucket">>, K, ?STD_TAG) + end, ObjL2), - {async, BucketFolder} = book_returnfolder(Bookie1, - {bucket_stats, "Bucket"}), + {async, BucketFolder} = + book_returnfolder(Bookie1, {bucket_stats, <<"Bucket">>}), {_Size, Count} = BucketFolder(), ?assertMatch(100, Count), FoldKeysFun = fun(_B, Item, FKFAcc) -> FKFAcc ++ [Item] end, - {async, - IndexFolder} = book_returnfolder(Bookie1, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f8", "f9"}, - {false, undefined}}), + {async, IndexFolder} = + book_returnfolder( + Bookie1, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f8">>, <<"f9">>}, + {false, undefined} + } + ), KeyList = IndexFolder(), ?assertMatch(20, length(KeyList)), {ok, Regex} = re:compile("f8"), - {async, - IndexFolderTR} = book_returnfolder(Bookie1, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f8", "f9"}, - {true, Regex}}), + {async, IndexFolderTR} = + book_returnfolder( + Bookie1, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f8">>, <<"f9">>}, + {true, Regex}} + ), TermKeyList = IndexFolderTR(), ?assertMatch(10, length(TermKeyList)), ok = book_close(Bookie1), {ok, Bookie2} = book_start([{root_path, RootPath}]), - {async, - IndexFolderTR2} = book_returnfolder(Bookie2, - {index_query, - "Bucket", - {FoldKeysFun, []}, - {"idx1_bin", "f7", "f9"}, - {false, Regex}}), + {async, IndexFolderTR2} = + book_returnfolder( + Bookie2, + { + index_query, + <<"Bucket">>, + {FoldKeysFun, []}, + {<<"idx1_bin">>, <<"f7">>, <<"f9">>}, + {false, Regex}} + ), KeyList2 = IndexFolderTR2(), ?assertMatch(10, length(KeyList2)), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_get(Bookie2, "Bucket", K, ?STD_TAG) - end, - ObjL2), - lists:foreach(fun({K, _V, _S}) -> - not_found = book_head(Bookie2, "Bucket", K, ?STD_TAG) - end, - ObjL2), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_get(Bookie2, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2 + ), + lists:foreach( + fun({K, _V, _S}) -> + not_found = book_head(Bookie2, <<"Bucket">>, K, ?STD_TAG) + end, + ObjL2 + ), ok = book_close(Bookie2), reset_filestructure(). @@ -2748,45 +2850,55 @@ hashlist_query_test_() -> hashlist_query_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ] + ), ObjL1 = generate_multiple_objects(1200, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), ObjL2 = generate_multiple_objects(20, 1201), % Put in a few objects with a TTL in the past Past = leveled_util:integer_now() - 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Past) end, - ObjL2), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Past) + end, + ObjL2 + ), % Scan the store for the Bucket, Keys and Hashes - {async, HTFolder} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + {async, HTFolder} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList = HTFolder(), - lists:foreach(fun({B, _K, H}) -> - ?assertMatch("Bucket", B), - ?assertMatch(true, is_integer(H)) - end, - KeyHashList), + lists:foreach( + fun({B, _K, H}) -> + ?assertMatch(<<"Bucket">>, B), + ?assertMatch(true, is_integer(H)) + end, + KeyHashList) + , ?assertMatch(1200, length(KeyHashList)), ok = book_close(Bookie1), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 200000}, - {cache_size, 500}]), - {async, HTFolder2} = book_returnfolder(Bookie2, - {hashlist_query, - ?STD_TAG, - false}), + {ok, Bookie2} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 200000}, + {cache_size, 500} + ] + ), + {async, HTFolder2} = + book_returnfolder(Bookie2, {hashlist_query, ?STD_TAG, false}), L0 = length(KeyHashList), HTR2 = HTFolder2(), ?assertMatch(L0, length(HTR2)), @@ -2800,26 +2912,28 @@ hashlist_query_withjournalcheck_test_() -> hashlist_query_withjournalcheck_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ] + ), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - {async, HTFolder1} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + lists:foreach( + fun({K, V, S}) -> + ok = book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + {async, HTFolder1} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList = HTFolder1(), - {async, HTFolder2} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - true}), + {async, HTFolder2} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, true}), ?assertMatch(KeyHashList, HTFolder2()), ok = book_close(Bookie1), reset_filestructure(). @@ -2829,68 +2943,89 @@ foldobjects_vs_hashtree_test_() -> foldobjects_vs_hashtree_testto() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), ObjL1 = generate_multiple_objects(800, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "Bucket", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - {async, HTFolder1} = book_returnfolder(Bookie1, - {hashlist_query, - ?STD_TAG, - false}), + lists:foreach( + fun({K, V, S}) -> ok = + book_tempput(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + {async, HTFolder1} = + book_returnfolder(Bookie1, {hashlist_query, ?STD_TAG, false}), KeyHashList1 = lists:usort(HTFolder1()), - FoldObjectsFun = fun(B, K, V, Acc) -> - [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, - {async, HTFolder2} = book_returnfolder(Bookie1, - {foldobjects_allkeys, - ?STD_TAG, - FoldObjectsFun, - true}), + FoldObjectsFun = + fun(B, K, V, Acc) -> + [{B, K, erlang:phash2(term_to_binary(V))}|Acc] + end, + {async, HTFolder2} = + book_returnfolder( + Bookie1, + { + foldobjects_allkeys, + ?STD_TAG, + FoldObjectsFun, + true + } + ), KeyHashList2 = HTFolder2(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList2)), FoldHeadsFun = fun(B, K, ProxyV, Acc) -> - {proxy_object, - _MDBin, - _Size, - {FetchFun, Clone, JK}} = binary_to_term(ProxyV), + {proxy_object, _MDBin, _Size, {FetchFun, Clone, JK}} = + binary_to_term(ProxyV), V = FetchFun(Clone, JK), [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, {async, HTFolder3} = - book_returnfolder(Bookie1, - {foldheads_allkeys, - ?STD_TAG, - FoldHeadsFun, - true, true, false, false, false}), + book_returnfolder( + Bookie1, + { + foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun, + true, + true, + false, + false, + false + } + ), KeyHashList3 = HTFolder3(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList3)), FoldHeadsFun2 = fun(B, K, ProxyV, Acc) -> - {proxy_object, - MD, - _Size1, - _Fetcher} = binary_to_term(ProxyV), + {proxy_object, MD, _Size1, _Fetcher} = binary_to_term(ProxyV), {Hash, _Size0, _UserDefinedMD} = MD, [{B, K, Hash}|Acc] end, {async, HTFolder4} = - book_returnfolder(Bookie1, - {foldheads_allkeys, - ?STD_TAG, - FoldHeadsFun2, - false, false, false, false, false}), + book_returnfolder( + Bookie1, + { + foldheads_allkeys, + ?STD_TAG, + FoldHeadsFun2, + false, + false, + false, + false, + false + } + ), KeyHashList4 = HTFolder4(), ?assertMatch(KeyHashList1, lists:usort(KeyHashList4)), @@ -2908,138 +3043,207 @@ foldobjects_vs_foldheads_bybucket_testto() -> folder_cache_test(CacheSize) -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, CacheSize}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, CacheSize + }] + ), _ = book_returnactors(Bookie1), ObjL1 = generate_multiple_objects(400, 1), ObjL2 = generate_multiple_objects(400, 1), % Put in all the objects with a TTL in the future Future = leveled_util:integer_now() + 300, - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "BucketA", K, V, S, - ?STD_TAG, - Future) end, - ObjL1), - lists:foreach(fun({K, V, S}) -> ok = book_tempput(Bookie1, - "BucketB", K, V, S, - ?STD_TAG, - Future) end, + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"BucketA">>, K, V, S, ?STD_TAG, Future) + end, + ObjL1 + ), + lists:foreach( + fun({K, V, S}) -> + ok = + book_tempput(Bookie1, <<"BucketB">>, K, V, S, ?STD_TAG, Future) + end, ObjL2), - FoldObjectsFun = fun(B, K, V, Acc) -> - [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, + FoldObjectsFun = + fun(B, K, V, Acc) -> + [{B, K, erlang:phash2(term_to_binary(V))}|Acc] + end, {async, HTFolder1A} = - book_returnfolder(Bookie1, - {foldobjects_bybucket, - ?STD_TAG, - "BucketA", - all, - FoldObjectsFun, - false}), + book_returnfolder( + Bookie1, + { + foldobjects_bybucket, + ?STD_TAG, + <<"BucketA">>, + all, + FoldObjectsFun, + false + } + ), KeyHashList1A = HTFolder1A(), {async, HTFolder1B} = - book_returnfolder(Bookie1, - {foldobjects_bybucket, - ?STD_TAG, - "BucketB", - all, - FoldObjectsFun, - true}), + book_returnfolder( + Bookie1, + { + foldobjects_bybucket, + ?STD_TAG, + <<"BucketB">>, + all, + FoldObjectsFun, + true + } + ), KeyHashList1B = HTFolder1B(), - ?assertMatch(false, - lists:usort(KeyHashList1A) == lists:usort(KeyHashList1B)), + ?assertMatch( + false, + lists:usort(KeyHashList1A) == lists:usort(KeyHashList1B) + ), FoldHeadsFun = fun(B, K, ProxyV, Acc) -> - {proxy_object, - _MDBin, - _Size, - {FetchFun, Clone, JK}} = binary_to_term(ProxyV), + {proxy_object, _MDBin, _Size, {FetchFun, Clone, JK}} = + binary_to_term(ProxyV), V = FetchFun(Clone, JK), [{B, K, erlang:phash2(term_to_binary(V))}|Acc] end, {async, HTFolder2A} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketA", - all, - FoldHeadsFun, - true, true, - false, false, false}), - KeyHashList2A = HTFolder2A(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketA">>, + all, + FoldHeadsFun, + true, + true, + false, + false,false + } + ), + KeyHashList2A = return_list_result(HTFolder2A), {async, HTFolder2B} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - all, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2B = HTFolder2B(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + all, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2B = return_list_result(HTFolder2B), - ?assertMatch(true, - lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A)), - ?assertMatch(true, - lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B)), + ?assertMatch( + true, + lists:usort(KeyHashList1A) == lists:usort(KeyHashList2A) + ), + ?assertMatch( + true, + lists:usort(KeyHashList1B) == lists:usort(KeyHashList2B) + ), {async, HTFolder2C} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", <<"$all">>}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2C = HTFolder2C(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, <<"$all">>}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2C = return_list_result(HTFolder2C), {async, HTFolder2D} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", "Keyzzzzz"}, - FoldHeadsFun, - true, true, - false, false, false}), - KeyHashList2D = HTFolder2D(), - ?assertMatch(true, - lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C)), - ?assertMatch(true, - lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D)), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, <<"Keyzzzzz">>}, + FoldHeadsFun, + true, + true, + false, + false, + false + } + ), + KeyHashList2D = return_list_result(HTFolder2D), + ?assertMatch( + true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2C) + ), + ?assertMatch( + true, + lists:usort(KeyHashList2B) == lists:usort(KeyHashList2D) + ), - CheckSplitQueryFun = fun(SplitInt) -> io:format("Testing SplitInt ~w~n", [SplitInt]), - SplitIntEnd = "Key" ++ integer_to_list(SplitInt) ++ "|", - SplitIntStart = "Key" ++ integer_to_list(SplitInt + 1), + SplitIntEnd = + list_to_binary("Key" ++ integer_to_list(SplitInt) ++ "|"), + SplitIntStart = + list_to_binary("Key" ++ integer_to_list(SplitInt + 1)), {async, HTFolder2E} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {"Key", SplitIntEnd}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2E = HTFolder2E(), + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {<<"Key">>, SplitIntEnd}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2E = return_list_result(HTFolder2E), {async, HTFolder2F} = - book_returnfolder(Bookie1, - {foldheads_bybucket, - ?STD_TAG, - "BucketB", - {SplitIntStart, "Key|"}, - FoldHeadsFun, - true, false, - false, false, false}), - KeyHashList2F = HTFolder2F(), - + book_returnfolder( + Bookie1, + { + foldheads_bybucket, + ?STD_TAG, + <<"BucketB">>, + {SplitIntStart, <<"Key|">>}, + FoldHeadsFun, + true, + false, + false, + false, + false + } + ), + KeyHashList2F = return_list_result(HTFolder2F), + ?assertMatch(true, length(KeyHashList2E) > 0), ?assertMatch(true, length(KeyHashList2F) > 0), + io:format("Length of 2B ~w 2E ~w 2F ~w~n", [length(KeyHashList2B), length(KeyHashList2E), @@ -3053,25 +3257,41 @@ folder_cache_test(CacheSize) -> ok = book_close(Bookie1), reset_filestructure(). +-spec return_list_result(fun(() -> list(dynamic()))) -> list(). +return_list_result(FoldFun) -> + case FoldFun() of + HL when is_list(HL) -> + HL + end. + small_cachesize_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 1}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 1} + ]), ok = leveled_bookie:book_close(Bookie1). is_empty_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [{ + root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), % Put in an object with a TTL in the future Future = leveled_util:integer_now() + 300, ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), - ok = book_tempput(Bookie1, - <<"B">>, <<"K">>, {value, <<"V">>}, [], - ?STD_TAG, Future), + ok = + book_tempput( + Bookie1, <<"B">>, <<"K">>, {value, <<"V">>}, [], ?STD_TAG, Future + ), ?assertMatch(false, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?RIAK_TAG)), @@ -3079,13 +3299,17 @@ is_empty_test() -> is_empty_headonly_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup} + ]), ?assertMatch(true, book_isempty(Bookie1, ?HEAD_TAG)), ObjSpecs = - [{add, <<"B1">>, <<"K1">>, <<1:8/integer>>, 100}, + [{add, <<"B1">>, <<"K1">>, <<1:8/integer>>, {size, 100}}, {remove, <<"B1">>, <<"K1">>, <<0:8/integer>>, null}], ok = book_mput(Bookie1, ObjSpecs), ?assertMatch(false, book_isempty(Bookie1, ?HEAD_TAG)), @@ -3098,30 +3322,32 @@ undefined_rootpath_test() -> ?assertMatch({error, no_root_path}, R), error_logger:tty(true). - foldkeys_headonly_test() -> - foldkeys_headonly_tester(5000, 25, "BucketStr"), + foldkeys_headonly_tester(5000, 25, <<"BucketStr">>), foldkeys_headonly_tester(2000, 25, <<"B0">>). - foldkeys_headonly_tester(ObjectCount, BlockSize, BStr) -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie1} = + book_start( + [{root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup}] + ), GenObjSpecFun = fun(I) -> Key = I rem 6, - {add, BStr, <>, integer_to_list(I), I} + {add, BStr, <>, <>, null} end, ObjSpecs = lists:map(GenObjSpecFun, lists:seq(1, ObjectCount)), ObjSpecBlocks = - lists:map(fun(I) -> - lists:sublist(ObjSpecs, I * BlockSize + 1, BlockSize) - end, - lists:seq(0, ObjectCount div BlockSize - 1)), + lists:map( + fun(I) -> + lists:sublist(ObjSpecs, I * BlockSize + 1, BlockSize) + end, + lists:seq(0, ObjectCount div BlockSize - 1)), lists:map(fun(Block) -> book_mput(Bookie1, Block) end, ObjSpecBlocks), ?assertMatch(false, book_isempty(Bookie1, ?HEAD_TAG)), @@ -3130,98 +3356,101 @@ foldkeys_headonly_tester(ObjectCount, BlockSize, BStr) -> ?HEAD_TAG, BStr, {fun(_B, {K, SK}, Acc) -> [{K, SK}|Acc] end, []} }, - {async, Folder1} = book_returnfolder(Bookie1, FolderT), - Key_SKL1 = lists:reverse(Folder1()), + Key_SKL_Compare = - lists:usort(lists:map(fun({add, _B, K, SK, _V}) -> {K, SK} end, ObjSpecs)), - ?assertMatch(Key_SKL_Compare, Key_SKL1), + lists:usort( + lists:map( + fun({add, _B, K, SK, _V}) -> {K, SK} end, ObjSpecs + ) + ), + {async, Folder1} = book_returnfolder(Bookie1, FolderT), + case Folder1() of + Key_SKL1 when is_list(Key_SKL1) -> + ?assertMatch(Key_SKL_Compare, lists:reverse(Key_SKL1)) + end, ok = book_close(Bookie1), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}, - {head_only, no_lookup}]), + {ok, Bookie2} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}, + {head_only, no_lookup} + ]), + {async, Folder2} = book_returnfolder(Bookie2, FolderT), - Key_SKL2 = lists:reverse(Folder2()), - ?assertMatch(Key_SKL_Compare, Key_SKL2), + case Folder2() of + Key_SKL2 when is_list(Key_SKL2) -> + ?assertMatch(Key_SKL_Compare, lists:reverse(Key_SKL2)) + end, ok = book_close(Bookie2). is_empty_stringkey_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), ?assertMatch(true, book_isempty(Bookie1, ?STD_TAG)), Past = leveled_util:integer_now() - 300, ?assertMatch(true, leveled_bookie:book_isempty(Bookie1, ?STD_TAG)), - ok = book_tempput(Bookie1, - "B", "K", {value, <<"V">>}, [], - ?STD_TAG, Past), - ok = book_put(Bookie1, - "B", "K0", {value, <<"V">>}, [], - ?STD_TAG), + ok = + book_tempput( + Bookie1, <<"B">>, <<"K">>, {value, <<"V">>}, [], ?STD_TAG, Past + ), + ok = book_put(Bookie1, <<"B">>, <<"K0">>, {value, <<"V">>}, [], ?STD_TAG), ?assertMatch(false, book_isempty(Bookie1, ?STD_TAG)), ok = book_close(Bookie1). scan_table_test() -> - K1 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K1">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA1">>), - K2 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K2">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA1">>), - K3 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K3">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AB1">>), - K4 = leveled_codec:to_ledgerkey(<<"B1">>, - <<"K4">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA2">>), - K5 = leveled_codec:to_ledgerkey(<<"B2">>, - <<"K5">>, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA2">>), + K1 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K1">>, ?IDX_TAG, <<"F1-bin">>, <<"AA1">>), + K2 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K2">>, ?IDX_TAG, <<"F1-bin">>, <<"AA1">>), + K3 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K3">>, ?IDX_TAG, <<"F1-bin">>, <<"AB1">>), + K4 = + leveled_codec:to_objectkey( + <<"B1">>, <<"K4">>, ?IDX_TAG, <<"F1-bin">>, <<"AA2">>), + K5 = + leveled_codec:to_objectkey( + <<"B2">>, <<"K5">>, ?IDX_TAG, <<"F1-bin">>, <<"AA2">>), Tab0 = ets:new(mem, [ordered_set]), - SK_A0 = leveled_codec:to_ledgerkey(<<"B1">>, - null, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA0">>), - EK_A9 = leveled_codec:to_ledgerkey(<<"B1">>, - null, - ?IDX_TAG, - <<"F1-bin">>, - <<"AA9">>), + SK_A0 = + leveled_codec:to_querykey( + <<"B1">>, null, ?IDX_TAG, <<"F1-bin">>, <<"AA0">>), + EK_A9 = + leveled_codec:to_querykey( + <<"B1">>, null, ?IDX_TAG, <<"F1-bin">>, <<"AA9">>), Empty = {[], infinity, 0}, - ?assertMatch(Empty, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch(Empty, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K1, {1, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}], 1, 1}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}], 1, 1}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K2, {2, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K3, {3, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch({[{K1, _}, {K2, _}], 1, 2}, scan_table(Tab0, SK_A0, EK_A9)), ets:insert(Tab0, [{K4, {4, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}, {K4, _}], 1, 4}, - scan_table(Tab0, SK_A0, EK_A9)), + ?assertMatch( + {[{K1, _}, {K2, _}, {K4, _}], 1, 4}, + scan_table(Tab0, SK_A0, EK_A9) + ), ets:insert(Tab0, [{K5, {5, active, no_lookup, null}}]), - ?assertMatch({[{K1, _}, {K2, _}, {K4, _}], 1, 4}, - scan_table(Tab0, SK_A0, EK_A9)). + ?assertMatch( + {[{K1, _}, {K2, _}, {K4, _}], 1, 4}, + scan_table(Tab0, SK_A0, EK_A9) + ). longrunning_test() -> SW = os:timestamp(), @@ -3229,31 +3458,36 @@ longrunning_test() -> ok = maybe_longrunning(SW, put). coverage_cheat_test() -> - {noreply, _State0} = handle_info(timeout, #state{}), - {ok, _State1} = code_change(null, #state{}, null). + DummyState = #state{inker = list_to_pid("<0.101.0>")}, + {noreply, _State0} = handle_info(timeout, DummyState), + {ok, _State1} = code_change(null, DummyState, null). erase_journal_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 50000}, - {cache_size, 100}]), + {ok, Bookie1} = + book_start( + [ + {root_path, RootPath}, + {max_journalsize, 50000}, + {cache_size, 100} + ]), ObjL1 = generate_multiple_objects(500, 1), % Put in all the objects with a TTL in the future lists:foreach( fun({K, V, S}) -> - ok = book_put(Bookie1, "Bucket", K, V, S, ?STD_TAG) + ok = book_put(Bookie1, <<"Bucket">>, K, V, S, ?STD_TAG) end, ObjL1), lists:foreach( fun({K, V, _S}) -> - {ok, V} = book_get(Bookie1, "Bucket", K, ?STD_TAG) + {ok, V} = book_get(Bookie1, <<"Bucket">>, K, ?STD_TAG) end, ObjL1), CheckHeadFun = fun(Book) -> fun({K, _V, _S}, Acc) -> - case book_head(Book, "Bucket", K, ?STD_TAG) of + case book_head(Book, <<"Bucket">>, K, ?STD_TAG) of {ok, _Head} -> Acc; not_found -> Acc + 1 end @@ -3265,70 +3499,73 @@ erase_journal_test() -> ok = book_close(Bookie1), io:format("Bookie closed - clearing Journal~n"), leveled_inker:clean_testdir(RootPath ++ "/" ++ ?JOURNAL_FP), - {ok, Bookie2} = book_start([{root_path, RootPath}, - {max_journalsize, 5000}, - {cache_size, 100}]), + {ok, Bookie2} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 5000}, + {cache_size, 100}] + ), HeadsNotFound2 = lists:foldl(CheckHeadFun(Bookie2), 0, ObjL1), ?assertMatch(500, HeadsNotFound2), ok = book_destroy(Bookie2). sqnorder_fold_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V1">>}, [], - ?STD_TAG), - ok = book_put(Bookie1, - <<"B">>, <<"K2">>, {value, <<"V2">>}, [], - ?STD_TAG), + {ok, Bookie1} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500}] + ), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V1">>}, [], ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K2">>, {value, <<"V2">>}, [], ?STD_TAG), FoldObjectsFun = fun(B, K, V, Acc) -> Acc ++ [{B, K, V}] end, {async, ObjFPre} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), {async, ObjFPost} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), - ok = book_put(Bookie1, - <<"B">>, <<"K3">>, {value, <<"V3">>}, [], - ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K3">>, {value, <<"V3">>}, [], ?STD_TAG), ObjLPre = ObjFPre(), - ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V1">>}}, - {<<"B">>, <<"K2">>, {value, <<"V2">>}}], ObjLPre), + ?assertMatch( + [{<<"B">>, <<"K1">>, {value, <<"V1">>}}, + {<<"B">>, <<"K2">>, {value, <<"V2">>}}], + ObjLPre + ), ObjLPost = ObjFPost(), - ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V1">>}}, - {<<"B">>, <<"K2">>, {value, <<"V2">>}}, - {<<"B">>, <<"K3">>, {value, <<"V3">>}}], ObjLPost), + ?assertMatch( + [{<<"B">>, <<"K1">>, {value, <<"V1">>}}, + {<<"B">>, <<"K2">>, {value, <<"V2">>}}, + {<<"B">>, <<"K3">>, {value, <<"V3">>}}], + ObjLPost + ), ok = book_destroy(Bookie1). sqnorder_mutatefold_test() -> RootPath = reset_filestructure(), - {ok, Bookie1} = book_start([{root_path, RootPath}, - {max_journalsize, 1000000}, - {cache_size, 500}]), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V1">>}, [], - ?STD_TAG), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V2">>}, [], - ?STD_TAG), + {ok, Bookie1} = + book_start([ + {root_path, RootPath}, + {max_journalsize, 1000000}, + {cache_size, 500} + ]), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V1">>}, [], ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V2">>}, [], ?STD_TAG), FoldObjectsFun = fun(B, K, V, Acc) -> Acc ++ [{B, K, V}] end, {async, ObjFPre} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, true, sqn_order), {async, ObjFPost} = - book_objectfold(Bookie1, - ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), + book_objectfold( + Bookie1, ?STD_TAG, {FoldObjectsFun, []}, false, sqn_order), - ok = book_put(Bookie1, - <<"B">>, <<"K1">>, {value, <<"V3">>}, [], - ?STD_TAG), + ok = book_put(Bookie1, <<"B">>, <<"K1">>, {value, <<"V3">>}, [], ?STD_TAG), ObjLPre = ObjFPre(), ?assertMatch([{<<"B">>, <<"K1">>, {value, <<"V2">>}}], ObjLPre), @@ -3340,19 +3577,22 @@ sqnorder_mutatefold_test() -> check_notfound_test() -> ProbablyFun = fun() -> probably end, MissingFun = fun() -> missing end, - MinFreq = lists:foldl(fun(_I, Freq) -> - {false, Freq0} = - check_notfound(Freq, ProbablyFun), - Freq0 - end, - 100, - lists:seq(1, 5000)), - % 5000 as needs to be a lot as doesn't decrement - % when random interval is not hit + MinFreq = + lists:foldl( + fun(_I, Freq) -> + {false, Freq0} = check_notfound(Freq, ProbablyFun), + Freq0 + end, + 100, + lists:seq(1, 5000)), + % 5000 as needs to be a lot as doesn't decrement + % when random interval is not hit ?assertMatch(?MIN_KEYCHECK_FREQUENCY, MinFreq), - ?assertMatch({true, ?MAX_KEYCHECK_FREQUENCY}, - check_notfound(?MAX_KEYCHECK_FREQUENCY, MissingFun)), + ?assertMatch( + {true, ?MAX_KEYCHECK_FREQUENCY}, + check_notfound(?MAX_KEYCHECK_FREQUENCY, MissingFun) + ), ?assertMatch({false, 0}, check_notfound(0, MissingFun)). diff --git a/src/leveled_cdb.erl b/src/leveled_cdb.erl index 6c1f6a67..d53f8e92 100644 --- a/src/leveled_cdb.erl +++ b/src/leveled_cdb.erl @@ -94,15 +94,11 @@ hashtable_calc/2]). -define(DWORD_SIZE, 8). --define(WORD_SIZE, 4). -define(MAX_FILE_SIZE, 3221225472). -define(BINARY_MODE, false). -define(BASE_POSITION, 2048). -define(WRITE_OPS, [binary, raw, read, write]). --define(PENDING_ROLL_WAIT, 30). -define(DELETE_TIMEOUT, 10000). --define(TIMING_SAMPLECOUNTDOWN, 5000). --define(TIMING_SAMPLESIZE, 100). -define(GETPOS_FACTOR, 8). -define(MAX_OBJECT_SIZE, 1000000000). % 1GB but really should be much smaller than this @@ -111,18 +107,24 @@ -record(state, {hashtree, last_position :: integer() | undefined, + % defined when writing, not required once rolled last_key = empty, current_count = 0 :: non_neg_integer(), hash_index = {} :: tuple(), filename :: string() | undefined, - handle :: file:fd() | undefined, - max_size :: pos_integer() | undefined, - max_count :: pos_integer() | undefined, + % defined when starting + handle :: file:io_device() | undefined, + % defined when starting + max_size :: pos_integer(), + max_count :: pos_integer(), binary_mode = false :: boolean(), delete_point = 0 :: integer(), inker :: pid() | undefined, + % undefined until delete_pending deferred_delete = false :: boolean(), - waste_path :: string() | undefined, + waste_path :: string()|undefined, + % undefined has functional meaning + % - no sending to waste on delete sync_strategy = none, log_options = leveled_log:get_opts() :: leveled_log:log_options(), @@ -133,8 +135,11 @@ -type hashtable_index() :: tuple(). -type file_location() :: integer()|eof. -type filter_fun() :: - fun((any(), binary(), integer(), any(), fun((binary()) -> any())) -> - {stop|loop, any()}). + fun((any(), + binary(), + integer(), + term()|{term(), term()}, + fun((binary()) -> any())) -> {stop|loop, any()}). -export_type([filter_fun/0]). @@ -265,11 +270,11 @@ cdb_getpositions(Pid, SampleSize) -> cdb_getpositions_fromidx(Pid, FC, Index, Acc) end end, - RandFun = fun(X) -> {leveled_rand:uniform(), X} end, + RandFun = fun(X) -> {rand:uniform(), X} end, SeededL = lists:map(RandFun, lists:seq(0, 255)), SortedL = lists:keysort(1, SeededL), PosList0 = lists:foldl(FoldFun, [], SortedL), - P1 = leveled_rand:uniform(max(1, length(PosList0) - S0)), + P1 = rand:uniform(max(1, length(PosList0) - S0)), lists:sublist(lists:sort(PosList0), P1, S0) end. @@ -367,7 +372,7 @@ cdb_scan(Pid, FilterFun, InitAcc, StartPosition) -> {cdb_scan, FilterFun, InitAcc, StartPosition}, infinity). --spec cdb_lastkey(pid()) -> any(). +-spec cdb_lastkey(pid()) -> leveled_codec:journal_key()|empty. %% @doc %% Get the last key to be added to the file (which will have the highest %% sequence number) @@ -487,38 +492,49 @@ starting({call, From}, {open_reader, Filename, LastKey}, State) -> {next_state, reader, State0, [{reply, From, ok}, hibernate]}. -writer({call, From}, {get_kv, Key}, State) -> +writer( + {call, From}, {get_kv, Key}, State = #state{handle =IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode)}]}; -writer({call, From}, {key_check, Key}, State) -> +writer( + {call, From}, {key_check, Key}, State = #state{handle =IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode, loose_presence)}]}; -writer({call, From}, {put_kv, Key, Value, Sync}, State) -> +writer( + {call, From}, + {put_kv, Key, Value, Sync}, + State = #state{last_position = LP, handle = IO}) + when ?IS_DEF(last_position), ?IS_DEF(IO) -> NewCount = State#state.current_count + 1, case NewCount >= State#state.max_count of true -> {keep_state_and_data, [{reply, From, roll}]}; false -> - Result = put(State#state.handle, - Key, - Value, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size, - State#state.last_key == empty), + Result = + put( + IO, + Key, + Value, + {LP, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size, + State#state.last_key == empty + ), case Result of roll -> %% Key and value could not be written @@ -545,7 +561,11 @@ writer({call, From}, {put_kv, Key, Value, Sync}, State) -> end; writer({call, From}, {mput_kv, []}, _State) -> {keep_state_and_data, [{reply, From, ok}]}; -writer({call, From}, {mput_kv, KVList}, State) -> +writer( + {call, From}, + {mput_kv, KVList}, + State = #state{last_position = LP, handle = IO}) + when ?IS_DEF(last_position), ?IS_DEF(IO) -> NewCount = State#state.current_count + length(KVList), TooMany = NewCount >= State#state.max_count, NotEmpty = State#state.current_count > 0, @@ -553,11 +573,14 @@ writer({call, From}, {mput_kv, KVList}, State) -> true -> {keep_state_and_data, [{reply, From, roll}]}; false -> - Result = mput(State#state.handle, - KVList, - {State#state.last_position, State#state.hashtree}, - State#state.binary_mode, - State#state.max_size), + Result = + mput( + IO, + KVList, + {LP, State#state.hashtree}, + State#state.binary_mode, + State#state.max_size + ), case Result of roll -> %% Keys and values could not be written @@ -573,38 +596,46 @@ writer({call, From}, {mput_kv, KVList}, State) -> [{reply, From, ok}]} end end; -writer({call, From}, cdb_complete, State) -> - NewName = determine_new_filename(State#state.filename), +writer( + {call, From}, cdb_complete, State = #state{filename = FN}) + when ?IS_DEF(FN) -> + NewName = determine_new_filename(FN), ok = close_file(State#state.handle, State#state.hashtree, State#state.last_position), - ok = rename_for_read(State#state.filename, NewName), + ok = rename_for_read(FN, NewName), {stop_and_reply, normal, [{reply, From, {ok, NewName}}]}; writer({call, From}, Event, State) -> handle_sync_event(Event, From, State); -writer(cast, cdb_roll, State) -> +writer( + cast, cdb_roll, State = #state{last_position = LP}) + when ?IS_DEF(LP) -> ok = leveled_iclerk:clerk_hashtablecalc( - State#state.hashtree, State#state.last_position, self()), + State#state.hashtree, LP, self()), {next_state, rolling, State}. -rolling({call, From}, {get_kv, Key}, State) -> +rolling( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode)}]}; -rolling({call, From}, {key_check, Key}, State) -> +rolling( + {call, From}, {key_check, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> {keep_state_and_data, [{reply, From, get_mem( Key, - State#state.handle, + IO, State#state.hashtree, State#state.binary_mode, loose_presence)}]}; @@ -612,15 +643,19 @@ rolling({call, From}, {get_positions, _SampleSize, _Index, SampleAcc}, _State) -> {keep_state_and_data, [{reply, From, SampleAcc}]}; -rolling({call, From}, {return_hashtable, IndexList, HashTreeBin}, State) -> +rolling( + {call, From}, + {return_hashtable, IndexList, HashTreeBin}, + State = #state{filename = FN}) + when ?IS_DEF(FN) -> SW = os:timestamp(), Handle = State#state.handle, {ok, BasePos} = file:position(Handle, State#state.last_position), - NewName = determine_new_filename(State#state.filename), + NewName = determine_new_filename(FN), ok = perform_write_hash_tables(Handle, HashTreeBin, BasePos), ok = write_top_index_table(Handle, BasePos, IndexList), file:close(Handle), - ok = rename_for_read(State#state.filename, NewName), + ok = rename_for_read(FN, NewName), leveled_log:log(cdb03, [NewName]), ets:delete(State#state.hashtree), {NewHandle, Index, LastKey} = @@ -646,13 +681,17 @@ rolling(cast, {delete_pending, ManSQN, Inker}, State) -> {keep_state, State#state{delete_point=ManSQN, inker=Inker, deferred_delete=true}}. -reader({call, From}, {get_kv, Key}, State) -> +reader( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - State#state.binary_mode, - State#state.monitor), + get_withcache( + IO, + Key, + State#state.hash_index, + State#state.binary_mode, + State#state.monitor + ), {keep_state_and_data, [{reply, From, Result}]}; reader({call, From}, {key_check, Key}, State) -> Result = @@ -673,8 +712,11 @@ reader({call, From}, {get_positions, SampleSize, Index, Acc}, State) -> {keep_state_and_data, [{reply, From, lists:sublist(UpdAcc, SampleSize)}]} end; -reader({call, From}, {direct_fetch, PositionList, Info}, State) -> - H = State#state.handle, +reader( + {call, From}, + {direct_fetch, PositionList, Info}, + State = #state{handle = IO}) + when ?IS_DEF(IO) -> FilterFalseKey = fun(Tpl) -> case element(1, Tpl) of @@ -687,20 +729,23 @@ reader({call, From}, {direct_fetch, PositionList, Info}, State) -> case Info of key_only -> - FM = lists:filtermap( + FM = + lists:filtermap( fun(P) -> - FilterFalseKey(extract_key(H, P)) end, - PositionList), + FilterFalseKey(extract_key(IO, P)) + end, + PositionList + ), MapFun = fun(T) -> element(1, T) end, {keep_state_and_data, [{reply, From, lists:map(MapFun, FM)}]}; key_size -> - FilterFun = fun(P) -> FilterFalseKey(extract_key_size(H, P)) end, + FilterFun = fun(P) -> FilterFalseKey(extract_key_size(IO, P)) end, {keep_state_and_data, [{reply, From, lists:filtermap(FilterFun, PositionList)}]}; key_value_check -> BM = State#state.binary_mode, - MapFun = fun(P) -> extract_key_value_check(H, P, BM) end, + MapFun = fun(P) -> extract_key_value_check(IO, P, BM) end, % direct_fetch will occur in batches, so it doesn't make sense to % hibernate the process that is likely to be used again. However, % a significant amount of unused binary references may have @@ -709,12 +754,13 @@ reader({call, From}, {direct_fetch, PositionList, Info}, State) -> garbage_collect(), {keep_state_and_data, []} end; -reader({call, From}, cdb_complete, State) -> - leveled_log:log(cdb05, [State#state.filename, reader, cdb_ccomplete]), - ok = file:close(State#state.handle), - {stop_and_reply, - normal, - [{reply, From, {ok, State#state.filename}}], +reader( + {call, From}, cdb_complete, State = #state{filename = FN, handle = IO}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, reader, cdb_ccomplete]), + ok = file:close(IO), + {stop_and_reply, normal, + [{reply, From, {ok, FN}}], State#state{handle=undefined}}; reader({call, From}, check_hashtable, _State) -> {keep_state_and_data, [{reply, From, true}]}; @@ -731,69 +777,77 @@ reader(cast, clerk_complete, _State) -> {keep_state_and_data, [hibernate]}. -delete_pending({call, From}, {get_kv, Key}, State) -> +delete_pending( + {call, From}, {get_kv, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - State#state.binary_mode, - State#state.monitor), + get_withcache( + IO, + Key, + State#state.hash_index, + State#state.binary_mode, + State#state.monitor + ), {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, {key_check, Key}, State) -> +delete_pending( + {call, From}, {key_check, Key}, State = #state{handle = IO}) + when ?IS_DEF(IO) -> Result = - get_withcache(State#state.handle, - Key, - State#state.hash_index, - loose_presence, - State#state.binary_mode, - {no_monitor, 0}), + get_withcache( + IO, + Key, + State#state.hash_index, + loose_presence, + State#state.binary_mode, + {no_monitor, 0} + ), {keep_state_and_data, [{reply, From, Result}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, cdb_close, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, cdb_close]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), +delete_pending( + {call, From}, cdb_close, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, delete_pending, cdb_close]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop_and_reply, normal, [{reply, From, ok}]}; -delete_pending(cast, delete_confirmed, State=#state{delete_point=ManSQN}) -> - leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), +delete_pending( + cast, delete_confirmed, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb04, [FN, State#state.delete_point]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop, normal}; -delete_pending(cast, destroy, State) -> - leveled_log:log(cdb05, [State#state.filename, delete_pending, destroy]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), +delete_pending( + cast, destroy, State = #state{handle = IO, filename = FN}) + when ?IS_DEF(FN), ?IS_DEF(IO) -> + leveled_log:log(cdb05, [FN, delete_pending, destroy]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop, normal}; delete_pending( - timeout, _, State=#state{delete_point=ManSQN}) when ManSQN > 0 -> + timeout, _, State=#state{delete_point=ManSQN, handle = IO, filename = FN}) + when ManSQN > 0, ?IS_DEF(FN), ?IS_DEF(IO) -> case is_process_alive(State#state.inker) of true -> ok = - leveled_inker:ink_confirmdelete(State#state.inker, - ManSQN, - self()), + leveled_inker:ink_confirmdelete( + State#state.inker, ManSQN, self()), {keep_state_and_data, [?DELETE_TIMEOUT]}; false -> - leveled_log:log(cdb04, [State#state.filename, ManSQN]), - close_pendingdelete(State#state.handle, - State#state.filename, - State#state.waste_path), + leveled_log:log(cdb04, [FN, ManSQN]), + close_pendingdelete(IO, FN, State#state.waste_path), {stop, normal} end. -handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> - {ok, EndPos0} = file:position(State#state.handle, eof), +handle_sync_event( + {cdb_scan, FilterFun, Acc, StartPos}, From, State = #state{handle = IO}) + when ?IS_DEF(IO) -> + {ok, EndPos0} = file:position(IO, eof), {ok, StartPos0} = case StartPos of undefined -> - file:position(State#state.handle, ?BASE_POSITION); + file:position(IO, ?BASE_POSITION); StartPos -> {ok, StartPos} end, - file:position(State#state.handle, StartPos0), + file:position(IO, StartPos0), MaybeEnd = (check_last_key(State#state.last_key) == empty) or (StartPos0 >= (EndPos0 - ?DWORD_SIZE)), @@ -802,11 +856,13 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> true -> {eof, Acc}; false -> - scan_over_file(State#state.handle, - StartPos0, - FilterFun, - Acc, - State#state.last_key) + scan_over_file( + IO, + StartPos0, + FilterFun, + Acc, + State#state.last_key + ) end, % The scan may have created a lot of binary references, clear up the % reference counters for this process here manually. The cdb process @@ -821,20 +877,26 @@ handle_sync_event({cdb_scan, FilterFun, Acc, StartPos}, From, State) -> {keep_state_and_data, []}; handle_sync_event(cdb_lastkey, From, State) -> {keep_state_and_data, [{reply, From, State#state.last_key}]}; -handle_sync_event(cdb_firstkey, From, State) -> - {ok, EOFPos} = file:position(State#state.handle, eof), - FilterFun = fun(Key, _V, _P, _O, _Fun) -> {stop, Key} end, +handle_sync_event( + cdb_firstkey, From, State = #state{handle = IO}) + when ?IS_DEF(IO) -> + {ok, EOFPos} = file:position(IO, eof), FirstKey = case EOFPos of ?BASE_POSITION -> empty; _ -> - file:position(State#state.handle, ?BASE_POSITION), - {_Pos, FirstScanKey} = scan_over_file(State#state.handle, - ?BASE_POSITION, - FilterFun, - empty, - State#state.last_key), + FindFirstKeyFun = + fun(Key, _V, _P, _O, _Fun) -> {stop, Key} end, + file:position(IO, ?BASE_POSITION), + {_Pos, FirstScanKey} = + scan_over_file( + IO, + ?BASE_POSITION, + FindFirstKeyFun, + empty, + State#state.last_key + ), FirstScanKey end, {keep_state_and_data, [{reply, From, FirstKey}]}; @@ -861,14 +923,15 @@ handle_sync_event({put_cachedscore, Score}, From, State) -> {keep_state, State#state{cached_score = {Score,os:timestamp()}}, [{reply, From, ok}]}; -handle_sync_event(cdb_close, From, State) -> - file:close(State#state.handle), +handle_sync_event( + cdb_close, From, _State = #state{handle = IO}) + when ?IS_DEF(IO) -> + file:close(IO), {stop_and_reply, normal, [{reply, From, ok}]}. terminate(_Reason, _StateName, _State) -> ok. - code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. @@ -877,7 +940,6 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%% External functions %%%============================================================================ - finished_rolling(CDB) -> RollerFun = fun(Sleep, FinishedRolling) -> @@ -908,9 +970,9 @@ close_pendingdelete(Handle, Filename, WasteFP) -> undefined -> ok = file:delete(Filename); WasteFP -> - Components = filename:split(Filename), - NewName = WasteFP ++ lists:last(Components), - file:rename(Filename, NewName) + FN = filename:basename(Filename), + NewName = filename:join(WasteFP, FN), + ok = file:rename(Filename, NewName) end; false -> % This may happen when there has been a destroy while files are @@ -1179,7 +1241,7 @@ find_lastkey(Handle, IndexCache) -> _ -> {ok, _} = file:position(Handle, LastPosition), {KeyLength, _ValueLength} = read_next_2_integers(Handle), - safe_read_next_key(Handle, KeyLength) + safe_read_next(Handle, KeyLength, key) end. @@ -1239,7 +1301,7 @@ extract_kvpair(_H, [], _K, _BinaryMode) -> extract_kvpair(Handle, [Position|Rest], Key, BinaryMode) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - case safe_read_next_keybin(Handle, KeyLength) of + case safe_read_next(Handle, KeyLength, keybin) of {Key, KeyBin} -> % If same key as passed in, then found! case checkread_next_value(Handle, ValueLength, KeyBin) of {false, _} -> @@ -1259,12 +1321,12 @@ extract_kvpair(Handle, [Position|Rest], Key, BinaryMode) -> extract_key(Handle, Position) -> {ok, _} = file:position(Handle, Position), {KeyLength, _ValueLength} = read_next_2_integers(Handle), - {safe_read_next_key(Handle, KeyLength)}. + {safe_read_next(Handle, KeyLength, key)}. extract_key_size(Handle, Position) -> {ok, _} = file:position(Handle, Position), {KeyLength, ValueLength} = read_next_2_integers(Handle), - K = safe_read_next_key(Handle, KeyLength), + K = safe_read_next(Handle, KeyLength, key), {K, ValueLength}. extract_key_value_check(Handle, Position, BinaryMode) -> @@ -1279,32 +1341,35 @@ extract_key_value_check(Handle, Position, BinaryMode) -> end. --spec startup_scan_over_file(file:io_device(), file_location()) - -> {file_location(), any()}. +-spec startup_scan_over_file( + file:io_device(), integer()) -> {integer(), {ets:tid(), term()}}. %% @doc %% Scan through the file until there is a failure to crc check an input, and %% at that point return the position and the key dictionary scanned so far startup_scan_over_file(Handle, Position) -> - HashTree = new_hashtree(), - {eof, Output} = scan_over_file(Handle, - Position, - fun startup_filter/5, - {HashTree, empty}, - empty), + Hashtree = new_hashtree(), + FilterFun = startup_filter(Hashtree), + {eof, LastKey} = scan_over_file(Handle, Position, FilterFun, empty, empty), {ok, FinalPos} = file:position(Handle, cur), - {FinalPos, Output}. - + {FinalPos, {Hashtree, LastKey}}. +-spec startup_filter(ets:tid()) -> filter_fun(). %% @doc %% Specific filter to be used at startup to build a hashtree for an incomplete %% cdb file, and returns at the end the hashtree and the final Key seen in the %% journal -startup_filter(Key, _ValueAsBin, Position, {Hashtree, _LastKey}, _ExtractFun) -> - {loop, {put_hashtree(Key, Position, Hashtree), Key}}. +startup_filter(Hashtree) -> + FilterFun = + fun(Key, _ValueAsBin, Position, _LastKey, _ExtractFun) -> + put_hashtree(Key, Position, Hashtree), + {loop, Key} + end, + FilterFun. --spec scan_over_file(file:io_device(), file_location(), - filter_fun(), any(), any()) -> {file_location(), any()}. +-spec scan_over_file + (file:io_device(), integer(), filter_fun(), term(), any()) -> + {file_location(), term()}. %% Scan for key changes - scan over file returning applying FilterFun %% The FilterFun should accept as input: %% - Key, ValueBin, Position, Accumulator, Fun (to extract values from Binary) @@ -1324,13 +1389,14 @@ scan_over_file(Handle, Position, FilterFun, Output, LastKey) -> {ok, Position} = file:position(Handle, {bof, Position}), {eof, Output}; {Key, ValueAsBin, KeyLength, ValueLength} -> - NewPosition = case Key of - LastKey -> - eof; - _ -> - Position + KeyLength + ValueLength - + ?DWORD_SIZE - end, + NewPosition = + case Key of + LastKey -> + eof; + _ -> + Position + KeyLength + ValueLength + + ?DWORD_SIZE + end, case FilterFun(Key, ValueAsBin, Position, @@ -1360,9 +1426,8 @@ check_last_key(empty) -> check_last_key(_LK) -> ok. - --spec saferead_keyvalue(file:io_device()) - -> false|{any(), any(), integer(), integer()}. +-spec saferead_keyvalue( + file:io_device()) -> false|{any(), binary(), integer(), integer()}. %% @doc %% Read the Key/Value at this point, returning {ok, Key, Value} %% catch expected exceptions associated with file corruption (or end) and @@ -1372,11 +1437,11 @@ saferead_keyvalue(Handle) -> eof -> false; {KeyL, ValueL} when is_integer(KeyL), is_integer(ValueL) -> - case safe_read_next_keybin(Handle, KeyL) of + case safe_read_next(Handle, KeyL, keybin) of false -> false; {Key, KeyBin} -> - case safe_read_next_value(Handle, ValueL, KeyBin) of + case safe_read_next(Handle, ValueL, {value, KeyBin}) of false -> false; TrueValue -> @@ -1388,66 +1453,37 @@ saferead_keyvalue(Handle) -> false end. - --spec safe_read_next_key(file:io_device(), integer()) -> false|term(). -%% @doc -%% Return the next key or have false returned if there is some sort of -%% potentially expected error (e.g. due to file truncation). Note that no -%% CRC check has been performed -safe_read_next_key(Handle, Length) -> - ReadFun = fun(Bin) -> binary_to_term(Bin) end, - safe_read_next(Handle, Length, ReadFun). - --spec safe_read_next_keybin(file:io_device(), integer()) - -> false|{term(), binary()}. -%% @doc -%% Return the next key or have false returned if there is some sort of -%% potentially expected error (e.g. due to file truncation). Note that no -%% CRC check has been performed -%% Returns both the Key and the Binary version, the binary version being -%% required for the CRC checking after the value fetch (see -%% safe_read_next_value/3) -safe_read_next_keybin(Handle, Length) -> - ReadFun = fun(Bin) -> {binary_to_term(Bin), Bin} end, - safe_read_next(Handle, Length, ReadFun). - --spec safe_read_next_value(file:io_device(), integer(), binary()) - -> binary()|false. -safe_read_next_value(Handle, Length, KeyBin) -> - ReadFun = fun(VBin) -> crccheck(VBin, KeyBin) end, - safe_read_next(Handle, Length, ReadFun). - --type read_output() :: {term(), binary()}|binary()|term()|false. --type read_fun() :: fun((binary()) -> read_output()). - --spec safe_read_next(file:io_device(), integer(), read_fun()) - -> read_output(). +-spec safe_read_next + (file:io_device(), integer(), key) -> false|term(); + (file:io_device(), integer(), keybin) -> false|{term(), binary()}; + (file:io_device(), integer(), {value, binary()}) -> false|binary(). %% @doc %% Read the next item of length Length %% Previously catching error:badarg was sufficient to capture errors of %% corruption, but on some OS versions may need to catch error:einval as well -safe_read_next(Handle, Length, ReadFun) -> +safe_read_next(Handle, Length, ReadType) -> + ReadFun = + case ReadType of + key -> + fun(Bin) -> binary_to_term(Bin) end; + keybin -> + fun(KBin) -> {binary_to_term(KBin), KBin} end; + {value, KeyBin} -> + fun(VBin) -> crccheck(VBin, KeyBin) end + end, try - loose_read(Handle, Length, ReadFun) + case file:read(Handle, Length) of + eof -> + false; + {ok, Result} -> + ReadFun(Result) + end catch error:ReadError -> leveled_log:log(cdb20, [ReadError, Length]), false end. --spec loose_read(file:io_device(), integer(), read_fun()) -> read_output(). -%% @doc -%% Read with minimal error handling (only eof) - to be wrapped in -%% safe_read_next/3 to catch exceptions. -loose_read(Handle, Length, ReadFun) -> - case file:read(Handle, Length) of - eof -> - false; - {ok, Result} -> - ReadFun(Result) - end. - - -spec crccheck(binary()|bitstring(), binary()) -> any(). %% @doc %% CRC chaeck the value which should be a binary, where the first four bytes @@ -1472,8 +1508,9 @@ crccheck(_V, _KB) -> calc_crc(KeyBin, Value) -> erlang:crc32(<>). --spec checkread_next_value(file:io_device(), integer(), binary()) - -> {boolean(), binary()|crc_wonky}. +-spec checkread_next_value + (file:io_device(), integer(), binary()) -> + {true, binary()}|{false, crc_wonky}. %% @doc %% Read next string where the string has a CRC prepended - stripping the crc %% and checking if requested @@ -1578,12 +1615,14 @@ search_hash_table(Handle, leveled_monitor:timing(), leveled_monitor:timing(), pos_integer()) -> ok. -maybelog_get_timing(_Monitor, no_timing, no_timing, _CC) -> - ok; -maybelog_get_timing({Pid, _StatsFreq}, IndexTime, ReadTime, CycleCount) -> +maybelog_get_timing( + {Pid, _StatsFreq}, IndexTime, ReadTime, CycleCount) + when is_pid(Pid), is_integer(IndexTime), is_integer(ReadTime) -> leveled_monitor:add_stat( - Pid, {cdb_get_update, CycleCount, IndexTime, ReadTime}). - + Pid, {cdb_get_update, CycleCount, IndexTime, ReadTime}); +maybelog_get_timing(_Monitor, _IndexTime, _ReadTime, _CC) -> + ok. + %% Write the actual hashtables at the bottom of the file. Each hash table %% entry is a doubleword in length. The first word is the hash value @@ -1916,7 +1955,7 @@ dump(FileName) -> {ok, _} = file:position(Handle, {bof, ?BASE_POSITION}), Fn1 = fun(_I, Acc) -> {KL, VL} = read_next_2_integers(Handle), - {Key, KB} = safe_read_next_keybin(Handle, KL), + {Key, KB} = safe_read_next(Handle, KL, keybin), Value = case checkread_next_value(Handle, VL, KB) of {true, V0} -> @@ -2632,7 +2671,7 @@ safe_read_test() -> {ok, HandleK} = file:open(TestFN, ?WRITE_OPS), ok = file:pwrite(HandleK, 0, BinToWrite), {ok, _} = file:position(HandleK, 8 + KeyL + ValueL), - ?assertMatch(false, safe_read_next_key(HandleK, KeyL)), + ?assertMatch(false, safe_read_next(HandleK, KeyL, key)), ok = file:close(HandleK), WrongKeyL = endian_flip(KeyL + ValueL), @@ -2749,11 +2788,15 @@ getpositions_sample_test() -> ok = cdb_close(P2), file:delete(F2). - nonsense_coverage_test() -> - ?assertMatch({ok, reader, #state{}}, code_change(nonsense, - reader, - #state{}, - nonsense)). + ?assertMatch( + {ok, reader, #state{}}, + code_change( + nonsense, + reader, + #state{max_count=1, max_size=100}, + nonsense + ) + ). -endif. diff --git a/src/leveled_codec.erl b/src/leveled_codec.erl index bb69d814..44e65ae0 100644 --- a/src/leveled_codec.erl +++ b/src/leveled_codec.erl @@ -10,6 +10,12 @@ -include("leveled.hrl"). +-eqwalizer({nowarn_function, convert_to_ledgerv/5}). + +-ifdef(TEST). +-export([convert_to_ledgerv/5]). +-endif. + -export([ inker_reload_strategy/1, strip_to_seqonly/1, @@ -21,8 +27,10 @@ endkey_passed/2, key_dominates/2, maybe_reap_expiredkey/2, - to_ledgerkey/3, - to_ledgerkey/5, + to_objectkey/3, + to_objectkey/5, + to_querykey/3, + to_querykey/5, from_ledgerkey/1, from_ledgerkey/2, isvalid_ledgerkey/1, @@ -33,11 +41,11 @@ from_journalkey/1, revert_to_keydeltas/2, is_full_journalentry/1, - split_inkvalue/1, check_forinkertype/2, get_tagstrategy/2, maybe_compress/2, create_value_for_journal/3, + revert_value_from_journal/1, generate_ledgerkv/5, get_size/2, get_keyandobjhash/2, @@ -54,8 +62,9 @@ -type tag() :: leveled_head:object_tag()|?IDX_TAG|?HEAD_TAG|atom(). --type key() :: - binary()|string()|{binary(), binary()}. +-type single_key() :: binary(). +-type tuple_key() :: {single_key(), single_key()}. +-type key() :: single_key()|tuple_key(). % Keys SHOULD be binary() % string() support is a legacy of old tests -type sqn() :: @@ -75,8 +84,15 @@ -type ledger_status() :: tomb|{active, non_neg_integer()|infinity}. +-type primary_key() :: + {leveled_head:object_tag(), key(), single_key(), single_key()|null}. + % Primary key for an object +-type object_key() :: + {tag(), key(), key(), single_key()|null}. +-type query_key() :: + {tag(), key()|null, key()|null, single_key()|null}|all. -type ledger_key() :: - {tag(), any(), any(), any()}|all. + object_key()|query_key(). -type slimmed_key() :: {binary(), binary()|null}|binary()|null|all. -type ledger_value() :: @@ -86,7 +102,7 @@ -type ledger_value_v2() :: {sqn(), ledger_status(), segment_hash(), metadata(), last_moddate()}. -type ledger_kv() :: - {ledger_key(), ledger_value()}. + {object_key(), ledger_value()}. -type compaction_method() :: retain|recovr|recalc. -type compaction_strategy() :: @@ -94,14 +110,14 @@ -type journal_key_tag() :: ?INKT_STND|?INKT_TOMB|?INKT_MPUT|?INKT_KEYD. -type journal_key() :: - {sqn(), journal_key_tag(), ledger_key()}. + {sqn(), journal_key_tag(), primary_key()}. -type journal_ref() :: - {ledger_key(), sqn()}. + {object_key(), sqn()}. -type object_spec_v0() :: - {add|remove, key(), key(), key()|null, any()}. + {add|remove, key(), single_key(), single_key()|null, metadata()}. -type object_spec_v1() :: - {add|remove, v1, key(), key(), key()|null, - list(erlang:timestamp())|undefined, any()}. + {add|remove, v1, key(), single_key(), single_key()|null, + list(erlang:timestamp())|undefined, metadata()}. -type object_spec() :: object_spec_v0()|object_spec_v1(). -type compression_method() :: @@ -135,10 +151,14 @@ -export_type([tag/0, key/0, + single_key/0, sqn/0, object_spec/0, segment_hash/0, ledger_status/0, + primary_key/0, + object_key/0, + query_key/0, ledger_key/0, ledger_value/0, ledger_kv/0, @@ -186,30 +206,26 @@ segment_hash(KeyTuple) when is_tuple(KeyTuple) -> segment_hash(BinKey). -headkey_to_canonicalbinary({?HEAD_TAG, Bucket, Key, SubK}) - when is_binary(Bucket), is_binary(Key), is_binary(SubK) -> +headkey_to_canonicalbinary({ + ?HEAD_TAG, Bucket, Key, SubK}) + when is_binary(Bucket), is_binary(Key), is_binary(SubK) -> <>; -headkey_to_canonicalbinary({?HEAD_TAG, Bucket, Key, null}) - when is_binary(Bucket), is_binary(Key) -> +headkey_to_canonicalbinary( + {?HEAD_TAG, Bucket, Key, null}) + when is_binary(Bucket), is_binary(Key) -> <>; -headkey_to_canonicalbinary({?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) - when is_binary(BucketType), is_binary(Bucket) -> - headkey_to_canonicalbinary({?HEAD_TAG, - <>, - Key, - SubKey}); -headkey_to_canonicalbinary(Key) when element(1, Key) == ?HEAD_TAG -> - % In unit tests head specs can have non-binary keys, so handle - % this through hashing the whole key - leveled_util:t2b(Key). - +headkey_to_canonicalbinary( + {?HEAD_TAG, {BucketType, Bucket}, Key, SubKey}) + when is_binary(BucketType), is_binary(Bucket) -> + headkey_to_canonicalbinary( + {?HEAD_TAG, <>, Key, SubKey}). -spec to_lookup(ledger_key()) -> maybe_lookup(). %% @doc %% Should it be possible to lookup a key in the merge tree. This is not true %% For keys that should only be read through range queries. Direct lookup %% keys will have presence in bloom filters and other lookup accelerators. -to_lookup(Key) -> +to_lookup(Key) when is_tuple(Key) -> case element(1, Key) of ?IDX_TAG -> no_lookup; @@ -235,12 +251,12 @@ strip_to_keyseqonly({LK, V}) -> {LK, element(1, V)}. -spec strip_to_indexdetails(ledger_kv()) -> {integer(), segment_hash(), last_moddate()}. -strip_to_indexdetails({_, V}) when tuple_size(V) == 4 -> +strip_to_indexdetails({_, {SQN, _, SegmentHash, _}}) -> % A v1 value - {element(1, V), element(3, V), undefined}; -strip_to_indexdetails({_, V}) when tuple_size(V) > 4 -> + {SQN, SegmentHash, undefined}; +strip_to_indexdetails({_, {SQN, _, SegmentHash, _, LMD}}) -> % A v2 value should have a fith element - Last Modified Date - {element(1, V), element(3, V), element(5, V)}. + {SQN, SegmentHash, LMD}. -spec striphead_to_v1details(ledger_value()) -> ledger_value(). striphead_to_v1details(V) -> @@ -292,18 +308,25 @@ maybe_accumulate( maybe_accumulate(T, Acc, Count, Filter, AccFun). -spec accumulate_index( - {boolean(), undefined|leveled_runner:mp()}, leveled_runner:acc_fun()) - -> any(). + {boolean(), undefined|leveled_runner:mp()}, + leveled_runner:fold_keys_fun()) + -> leveled_penciller:pclacc_fun(). accumulate_index({false, undefined}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, _IndexInfo, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, _IndexInfo, ObjKey}, _Value, Acc) + when ObjKey =/= null -> FoldKeysFun(Bucket, ObjKey, Acc) end; accumulate_index({true, undefined}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) + when IdxValue =/= null, ObjKey =/= null -> FoldKeysFun(Bucket, {IdxValue, ObjKey}, Acc) end; accumulate_index({AddTerm, TermRegex}, FoldKeysFun) -> - fun({?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) -> + fun( + {?IDX_TAG, Bucket, {_IdxFld, IdxValue}, ObjKey}, _Value, Acc) + when IdxValue =/= null, ObjKey =/= null -> case re:run(IdxValue, TermRegex) of nomatch -> Acc; @@ -343,17 +366,17 @@ maybe_reap(_, _) -> false. -spec count_tombs( - list(ledger_kv()), non_neg_integer()|not_counted) -> - non_neg_integer()|not_counted. -count_tombs(_List, not_counted) -> - not_counted; + list(ledger_kv()), non_neg_integer()) -> + non_neg_integer(). count_tombs([], Count) -> Count; -count_tombs([{_K, V}|T], Count) when element(2, V) == tomb -> - count_tombs(T, Count + 1); -count_tombs([_KV|T], Count) -> - count_tombs(T, Count). - +count_tombs([{_K, V}|T], Count) when is_tuple(V) -> + case element(2, V) of + tomb -> + count_tombs(T, Count + 1); + _ -> + count_tombs(T, Count) + end. -spec from_ledgerkey(atom(), tuple()) -> false|tuple(). %% @doc @@ -375,18 +398,37 @@ from_ledgerkey({?HEAD_TAG, Bucket, Key, SubKey}) -> from_ledgerkey({_Tag, Bucket, Key, _SubKey}) -> {Bucket, Key}. --spec to_ledgerkey(any(), any(), tag(), any(), any()) -> ledger_key(). +-spec to_objectkey( + key(), single_key(), tag(), binary(), binary()) -> object_key(). %% @doc %% Convert something into a ledger key -to_ledgerkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> +to_objectkey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> {?IDX_TAG, Bucket, {Field, Value}, Key}. --spec to_ledgerkey(any(), any(), tag()) -> ledger_key(). +-if(?OTP_RELEASE >= 26). +-spec to_objectkey + (key(), single_key(), leveled_head:object_tag()) -> primary_key(); + (key(), key(), tag()) -> object_key(). +-else. +-spec to_objectkey(key(), key()|single_key(), tag()) -> object_key(). +-endif. %% @doc %% Convert something into a ledger key -to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG) -> +to_objectkey(Bucket, {Key, SubKey}, ?HEAD_TAG) -> {?HEAD_TAG, Bucket, Key, SubKey}; -to_ledgerkey(Bucket, Key, Tag) -> +to_objectkey(Bucket, Key, Tag) -> + {Tag, Bucket, Key, null}. + +-spec to_querykey( + key(), single_key()|null, tag(), binary(), binary()) + -> query_key(). +to_querykey(Bucket, Key, Tag, Field, Value) when Tag == ?IDX_TAG -> + {?IDX_TAG, Bucket, {Field, Value}, Key}. + +-spec to_querykey(key()|null, key()|null, tag()) -> query_key(). +%% @doc +%% Convert something into a ledger query key +to_querykey(Bucket, Key, Tag) -> {Tag, Bucket, Key, null}. %% No spec - due to tests @@ -399,8 +441,8 @@ isvalid_ledgerkey(_LK) -> false. -spec endkey_passed( - ledger_key()|slimmed_key(), - ledger_key()|slimmed_key()) -> boolean(). + query_key()|slimmed_key(), + object_key()|slimmed_key()) -> boolean(). %% @doc %% Compare a key against a query key, only comparing elements that are non-null %% in the Query key. @@ -480,14 +522,19 @@ get_tagstrategy(Tag, Strategy) -> %%% Manipulate Journal Key and Value %%%============================================================================ --spec to_inkerkey(ledger_key(), non_neg_integer()) -> journal_key(). +-spec to_inkerkey(primary_key(), non_neg_integer()) -> journal_key(). %% @doc %% convertion from ledger_key to journal_key to allow for the key to be fetched to_inkerkey(LedgerKey, SQN) -> {SQN, ?INKT_STND, LedgerKey}. --spec to_inkerkv(ledger_key(), non_neg_integer(), any(), journal_keychanges(), - compression_method(), boolean()) -> {journal_key(), any()}. +-spec to_inkerkv( + primary_key(), + non_neg_integer(), + any(), + journal_keychanges(), + compression_method(), boolean()) + -> {journal_key(), binary()}. %% @doc %% Convert to the correct format of a Journal key and value to_inkerkv(LedgerKey, SQN, Object, KeyChanges, PressMethod, Compress) -> @@ -496,7 +543,7 @@ to_inkerkv(LedgerKey, SQN, Object, KeyChanges, PressMethod, Compress) -> create_value_for_journal({Object, KeyChanges}, Compress, PressMethod), {{SQN, InkerType, LedgerKey}, Value}. --spec revert_to_keydeltas(journal_key(), any()) -> {journal_key(), any()}. +-spec revert_to_keydeltas(journal_key(), binary()) -> {journal_key(), any()}. %% @doc %% If we wish to retain key deltas when an object in the Journal has been %% replaced - then this converts a Journal Key and Value into one which has no @@ -575,7 +622,7 @@ serialise_object(Object, false, _Method) -> serialise_object(Object, true, _Method) -> term_to_binary(Object, [compressed]). --spec revert_value_from_journal(binary()) -> {any(), journal_keychanges()}. +-spec revert_value_from_journal(binary()) -> {dynamic(), journal_keychanges()}. %% @doc %% Revert the object back to its deserialised state, along with the list of %% key changes associated with the change @@ -661,10 +708,6 @@ decode_valuetype(TypeInt) -> from_journalkey({SQN, _Type, LedgerKey}) -> {SQN, LedgerKey}. - -split_inkvalue(VBin) when is_binary(VBin) -> - revert_value_from_journal(VBin). - check_forinkertype(_LedgerKey, delete) -> ?INKT_TOMB; check_forinkertype(_LedgerKey, head_only) -> @@ -709,7 +752,7 @@ idx_indexspecs(IndexSpecs, Bucket, Key, SQN, TTL) -> gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL) -> Status = set_status(IdxOp, TTL), - {to_ledgerkey(Bucket, Key, ?IDX_TAG, IdxField, IdxTerm), + {to_objectkey(Bucket, Key, ?IDX_TAG, IdxField, IdxTerm), {SQN, Status, no_lookup, null}}. -spec gen_headspec(object_spec(), integer(), integer()|infinity) -> ledger_kv(). @@ -717,22 +760,30 @@ gen_indexspec(Bucket, Key, IdxOp, IdxField, IdxTerm, SQN, TTL) -> %% Take an object_spec as passed in a book_mput, and convert it into to a %% valid ledger key and value. Supports different shaped tuples for different %% versions of the object_spec -gen_headspec({IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL) -> +gen_headspec( + {IdxOp, v1, Bucket, Key, SubKey, LMD, Value}, SQN, TTL) + when is_binary(Key) -> % v1 object spec Status = set_status(IdxOp, TTL), - K = to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG), + K = + case SubKey of + null -> + to_objectkey(Bucket, Key, ?HEAD_TAG); + SKB when is_binary(SKB) -> + to_objectkey(Bucket, {Key, SKB}, ?HEAD_TAG) + end, {K, {SQN, Status, segment_hash(K), Value, get_last_lastmodification(LMD)}}; -gen_headspec({IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL) -> - % v0 object spec - Status = set_status(IdxOp, TTL), - K = to_ledgerkey(Bucket, {Key, SubKey}, ?HEAD_TAG), - {K, {SQN, Status, segment_hash(K), Value, undefined}}. +gen_headspec( + {IdxOp, Bucket, Key, SubKey, Value}, SQN, TTL) + when is_binary(Key) -> + gen_headspec({IdxOp, v1, Bucket, Key, SubKey, undefined, Value}, SQN, TTL). --spec return_proxy(leveled_head:object_tag()|leveled_head:headonly_tag(), - leveled_head:object_metadata(), - pid(), journal_ref()) - -> proxy_objectbin()|leveled_head:object_metadata(). +-spec return_proxy + (leveled_head:headonly_tag(), leveled_head:object_metadata(), null, journal_ref()) + -> leveled_head:object_metadata(); + (leveled_head:object_tag(), leveled_head:object_metadata(), pid(), journal_ref()) + -> proxy_objectbin(). %% @doc %% If the object has a value, return the metadata and a proxy through which %% the applictaion or runner can access the value. If it is a ?HEAD_TAG @@ -751,6 +802,9 @@ return_proxy(Tag, ObjMetadata, InkerClone, JournalRef) -> InkerClone, JournalRef}}). +-spec set_status( + add|remove, non_neg_integer()|infinity) -> + tomb|{active, non_neg_integer()|infinity}. set_status(add, TTL) -> {active, TTL}; set_status(remove, _TTL) -> @@ -758,10 +812,18 @@ set_status(remove, _TTL) -> tomb. -spec generate_ledgerkv( - tuple(), integer(), any(), integer(), tuple()|infinity) -> - {any(), any(), any(), - {{integer(), integer()}|no_lookup, integer()}, - list()}. + primary_key(), + integer(), + dynamic(), + integer(), + non_neg_integer()|infinity) -> + { + key(), + single_key(), + ledger_value_v2(), + {segment_hash(), non_neg_integer()|null}, + list(erlang:timestamp()) + }. %% @doc %% Function to extract from an object the information necessary to populate %% the Penciller's ledger. @@ -776,24 +838,22 @@ set_status(remove, _TTL) -> %% siblings) generate_ledgerkv(PrimaryKey, SQN, Obj, Size, TS) -> {Tag, Bucket, Key, _} = PrimaryKey, - Status = case Obj of - delete -> - tomb; - _ -> - {active, TS} - end, + Status = case Obj of delete -> tomb; _ -> {active, TS} end, Hash = segment_hash(PrimaryKey), {MD, LastMods} = leveled_head:extract_metadata(Tag, Size, Obj), ObjHash = leveled_head:get_hash(Tag, MD), - Value = {SQN, - Status, - Hash, - MD, - get_last_lastmodification(LastMods)}, + Value = + { + SQN, + Status, + Hash, + MD, + get_last_lastmodification(LastMods) + }, {Bucket, Key, Value, {Hash, ObjHash}, LastMods}. --spec get_last_lastmodification(list(erlang:timestamp())|undefined) - -> pos_integer()|undefined. +-spec get_last_lastmodification( + list(erlang:timestamp())|undefined) -> pos_integer()|undefined. %% @doc %% Get the highest of the last modifications measured in seconds. This will be %% stored as 4 bytes (unsigned) so will last for another 80 + years @@ -830,10 +890,10 @@ get_keyandobjhash(LK, Value) -> %% Get the next key to iterate from a given point next_key(Key) when is_binary(Key) -> <>; -next_key(Key) when is_list(Key) -> - Key ++ [0]; next_key({Type, Bucket}) when is_binary(Type), is_binary(Bucket) -> - {Type, next_key(Bucket)}. + UpdBucket = next_key(Bucket), + true = is_binary(UpdBucket), + {Type, UpdBucket}. %%%============================================================================ @@ -844,6 +904,17 @@ next_key({Type, Bucket}) when is_binary(Type), is_binary(Bucket) -> -include_lib("eunit/include/eunit.hrl"). +-spec convert_to_ledgerv( + leveled_codec:ledger_key(), + integer(), + any(), + integer(), + non_neg_integer()|infinity) -> leveled_codec:ledger_value(). +convert_to_ledgerv(PK, SQN, Obj, Size, TS) -> + {_B, _K, MV, _H, _LMs} = + leveled_codec:generate_ledgerkv(PK, SQN, Obj, Size, TS), + MV. + valid_ledgerkey_test() -> UserDefTag = {user_defined, <<"B">>, <<"K">>, null}, ?assertMatch(true, isvalid_ledgerkey(UserDefTag)), @@ -870,8 +941,8 @@ indexspecs_test() -> endkey_passed_test() -> TestKey = {i, null, null, null}, - K1 = {i, 123, {"a", "b"}, <<>>}, - K2 = {o, 123, {"a", "b"}, <<>>}, + K1 = {i, <<"123">>, {<<"a">>, <<"b">>}, <<>>}, + K2 = {o, <<"123">>, {<<"a">>, <<"b">>}, <<>>}, ?assertMatch(false, endkey_passed(TestKey, K1)), ?assertMatch(true, endkey_passed(TestKey, K2)). @@ -881,7 +952,7 @@ endkey_passed_test() -> %% Maybe 5 microseconds per hash hashperf_test() -> - OL = lists:map(fun(_X) -> leveled_rand:rand_bytes(8192) end, lists:seq(1, 1000)), + OL = lists:map(fun(_X) -> crypto:strong_rand_bytes(8192) end, lists:seq(1, 1000)), SW = os:timestamp(), _HL = lists:map(fun(Obj) -> erlang:phash2(Obj) end, OL), io:format(user, "1000 object hashes in ~w microseconds~n", @@ -899,8 +970,8 @@ head_segment_compare_test() -> headspec_v0v1_test() -> % A v0 object spec generates the same outcome as a v1 object spec with the % last modified date undefined - V1 = {add, v1, <<"B">>, <<"K">>, <<"SK">>, undefined, <<"V">>}, - V0 = {add, <<"B">>, <<"K">>, <<"SK">>, <<"V">>}, + V1 = {add, v1, <<"B">>, <<"K">>, <<"SK">>, undefined, {<<"V">>}}, + V0 = {add, <<"B">>, <<"K">>, <<"SK">>, {<<"V">>}}, TTL = infinity, ?assertMatch(true, gen_headspec(V0, 1, TTL) == gen_headspec(V1, 1, TTL)). diff --git a/src/leveled_ebloom.erl b/src/leveled_ebloom.erl index b74a5826..21a8f40b 100644 --- a/src/leveled_ebloom.erl +++ b/src/leveled_ebloom.erl @@ -91,8 +91,8 @@ check_hash({_SegHash, Hash}, BloomBin) when is_binary(BloomBin)-> list(leveled_codec:segment_hash()), tuple(), slot_count()) -> tuple(). map_hashes([], HashListTuple, _SlotCount) -> HashListTuple; -map_hashes([Hash|Rest], HashListTuple, SlotCount) -> - {Slot, [H0, H1]} = split_hash(element(2, Hash), SlotCount), +map_hashes([{_SH, EH}|Rest], HashListTuple, SlotCount) -> + {Slot, [H0, H1]} = split_hash(EH, SlotCount), SlotHL = element(Slot + 1, HashListTuple), map_hashes( Rest, @@ -174,11 +174,15 @@ generate_orderedkeys(Seqn, Count, Acc, BucketLow, BucketHigh) -> BucketExt = io_lib:format("K~4..0B", [BucketLow + BNumber]), KeyExt = - io_lib:format("K~8..0B", [Seqn * 100 + leveled_rand:uniform(100)]), - LK = leveled_codec:to_ledgerkey("Bucket" ++ BucketExt, "Key" ++ KeyExt, o), - Chunk = leveled_rand:rand_bytes(16), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity), + io_lib:format("K~8..0B", [Seqn * 100 + rand:uniform(100)]), + LK = + leveled_codec:to_objectkey( + list_to_binary("Bucket" ++ BucketExt), + list_to_binary("Key" ++ KeyExt), + o + ), + Chunk = crypto:strong_rand_bytes(16), + MV = leveled_codec:convert_to_ledgerv(LK, Seqn, Chunk, 64, infinity), generate_orderedkeys( Seqn + 1, Count - 1, [{LK, MV}|Acc], BucketLow, BucketHigh). @@ -236,7 +240,7 @@ test_bloom(N, Runs) -> fun(HashList) -> HitOrMissFun = fun (Entry, {HitL, MissL}) -> - case leveled_rand:uniform() < 0.5 of + case rand:uniform() < 0.5 of true -> {[Entry|HitL], MissL}; false -> diff --git a/src/leveled_head.erl b/src/leveled_head.erl index b4bd6a34..64c0e19c 100644 --- a/src/leveled_head.erl +++ b/src/leveled_head.erl @@ -82,7 +82,7 @@ -type appdefinable_headfun() :: fun((object_tag(), object_metadata()) -> head()). -type appdefinable_metadatafun() :: - fun(({leveled_codec:tag(), non_neg_integer(), any()}) -> + fun((leveled_codec:tag(), non_neg_integer(), binary()|delete) -> {object_metadata(), list(erlang:timestamp())}). -type appdefinable_indexspecsfun() :: fun((object_tag(), object_metadata(), object_metadata()|not_present) -> @@ -117,11 +117,13 @@ %% @doc %% Convert a key to a binary in a consistent way for the tag. The binary will %% then be used to create the hash -key_to_canonicalbinary({?RIAK_TAG, Bucket, Key, null}) - when is_binary(Bucket), is_binary(Key) -> +key_to_canonicalbinary( + {?RIAK_TAG, Bucket, Key, null}) + when is_binary(Bucket), is_binary(Key) -> <>; -key_to_canonicalbinary({?RIAK_TAG, {BucketType, Bucket}, Key, SubKey}) - when is_binary(BucketType), is_binary(Bucket) -> +key_to_canonicalbinary( + {?RIAK_TAG, {BucketType, Bucket}, Key, SubKey}) + when is_binary(BucketType), is_binary(Bucket) -> key_to_canonicalbinary({?RIAK_TAG, <>, Key, @@ -130,9 +132,11 @@ key_to_canonicalbinary(Key) when element(1, Key) == ?STD_TAG -> default_key_to_canonicalbinary(Key); key_to_canonicalbinary(Key) -> OverrideFun = - get_appdefined_function(key_to_canonicalbinary, - fun default_key_to_canonicalbinary/1, - 1), + get_appdefined_function( + key_to_canonicalbinary, + fun default_key_to_canonicalbinary/1, + 1 + ), OverrideFun(Key). default_key_to_canonicalbinary(Key) -> @@ -162,7 +166,7 @@ default_build_head(_Tag, Metadata) -> Metadata. --spec extract_metadata(object_tag(), non_neg_integer(), any()) +-spec extract_metadata(object_tag(), non_neg_integer(), binary()) -> {object_metadata(), list(erlang:timestamp())}. %% @doc %% Take the inbound object and extract from it the metadata to be stored within @@ -239,9 +243,8 @@ defined_objecttags() -> [?STD_TAG, ?RIAK_TAG]. --spec default_reload_strategy(object_tag()) - -> {object_tag(), - leveled_codec:compaction_method()}. +-spec default_reload_strategy( + object_tag()) -> {object_tag(), leveled_codec:compaction_method()}. %% @doc %% State the compaction_method to be used when reloading the Ledger from the %% journal for each object tag. Note, no compaction strategy required for @@ -249,25 +252,24 @@ defined_objecttags() -> default_reload_strategy(Tag) -> {Tag, retain}. - --spec get_size(object_tag()|headonly_tag(), object_metadata()) - -> non_neg_integer(). +-spec get_size( + object_tag()|headonly_tag(), object_metadata()) -> non_neg_integer(). %% @doc %% Fetch the size from the metadata -get_size(?RIAK_TAG, RiakObjectMetadata) -> - element(4, RiakObjectMetadata); -get_size(_Tag, ObjectMetadata) -> - element(2, ObjectMetadata). +get_size(?RIAK_TAG, {_, _, _, Size}) -> + Size; +get_size(_Tag, {_, Size, _}) -> + Size. --spec get_hash(object_tag()|headonly_tag(), object_metadata()) - -> non_neg_integer(). +-spec get_hash( + object_tag()|headonly_tag(), object_metadata()) -> non_neg_integer()|null. %% @doc %% Fetch the hash from the metadata -get_hash(?RIAK_TAG, RiakObjectMetadata) -> - element(3, RiakObjectMetadata); -get_hash(_Tag, ObjectMetadata) -> - element(1, ObjectMetadata). +get_hash(?RIAK_TAG, {_, _, Hash, _}) -> + Hash; +get_hash(_Tag, {Hash, _, _}) -> + Hash. -spec standard_hash(any()) -> non_neg_integer(). %% @doc @@ -280,9 +282,15 @@ standard_hash(Obj) -> %%% Handling Override Functions %%%============================================================================ --spec get_appdefined_function( - appdefinable_function(), appdefinable_function_fun(), non_neg_integer()) -> - appdefinable_function_fun(). +-spec get_appdefined_function + (key_to_canonicalbinary, appdefinable_keyfun(), 1) -> + appdefinable_keyfun(); + (build_head, appdefinable_headfun(), 2) -> + appdefinable_headfun(); + (extract_metadata, appdefinable_metadatafun(), 3) -> + appdefinable_metadatafun(); + (diff_indexspecs, appdefinable_indexspecsfun(), 3) -> + appdefinable_indexspecsfun(). %% @doc %% If a keylist of [{function_name, fun()}] has been set as an environment %% variable for a tag, then this FunctionName can be used instead of the @@ -300,8 +308,8 @@ get_appdefined_function(FunctionName, DefaultFun, RequiredArity) -> %%%============================================================================ --spec riak_extract_metadata(binary()|delete, non_neg_integer()) -> - {riak_metadata(), list()}. +-spec riak_extract_metadata( + binary()|delete, non_neg_integer()) -> {riak_metadata(), list()}. %% @doc %% Riak extract metadata should extract a metadata object which is a %% five-tuple of: diff --git a/src/leveled_iclerk.erl b/src/leveled_iclerk.erl index e2413484..a2f9d329 100644 --- a/src/leveled_iclerk.erl +++ b/src/leveled_iclerk.erl @@ -114,10 +114,10 @@ scoring_state :: scoring_state()|undefined, score_onein = 1 :: pos_integer()}). --record(candidate, {low_sqn :: integer() | undefined, - filename :: string() | undefined, - journal :: pid() | undefined, - compaction_perc :: float() | undefined}). +-record(candidate, {low_sqn :: integer(), + filename :: string(), + journal :: pid(), + compaction_perc :: float()}). -record(scoring_state, {filter_fun :: leveled_inker:filterfun(), filter_server :: leveled_inker:filterserver(), @@ -158,7 +158,13 @@ %% @doc %% Generate a new clerk clerk_new(InkerClerkOpts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), InkerClerkOpts], []). + {ok, Clerk} = + gen_server:start_link( + ?MODULE, + [leveled_log:get_opts(), InkerClerkOpts], + [] + ), + {ok, Clerk}. -spec clerk_compact(pid(), pid(), @@ -310,60 +316,71 @@ handle_cast({compact, Checker, InitiateFun, CloseFun, FilterFun, Manifest0}, end, ok = clerk_scorefilelist(self(), lists:filter(NotRollingFun, Manifest)), ScoringState = - #scoring_state{filter_fun = FilterFun, - filter_server = FilterServer, - max_sqn = MaxSQN, - close_fun = CloseFun, - start_time = SW}, + #scoring_state{ + filter_fun = FilterFun, + filter_server = FilterServer, + max_sqn = MaxSQN, + close_fun = CloseFun, + start_time = SW + }, {noreply, State#state{scored_files = [], scoring_state = ScoringState}}; -handle_cast({score_filelist, [Entry|Tail]}, State) -> +handle_cast( + {score_filelist, [Entry|Tail]}, + State = #state{scoring_state = ScoringState}) + when ?IS_DEF(ScoringState) -> Candidates = State#state.scored_files, {LowSQN, FN, JournalP, _LK} = Entry, - ScoringState = State#state.scoring_state, CpctPerc = case {leveled_cdb:cdb_getcachedscore(JournalP, os:timestamp()), - leveled_rand:uniform(State#state.score_onein) == 1, + rand:uniform(State#state.score_onein) == 1, State#state.score_onein} of {CachedScore, _UseNewScore, ScoreOneIn} when CachedScore == undefined; ScoreOneIn == 1 -> % If caches are not used, always use the current score - check_single_file(JournalP, - ScoringState#scoring_state.filter_fun, - ScoringState#scoring_state.filter_server, - ScoringState#scoring_state.max_sqn, - ?SAMPLE_SIZE, - ?BATCH_SIZE, - State#state.reload_strategy); + check_single_file( + JournalP, + ScoringState#scoring_state.filter_fun, + ScoringState#scoring_state.filter_server, + ScoringState#scoring_state.max_sqn, + ?SAMPLE_SIZE, + ?BATCH_SIZE, + State#state.reload_strategy + ); {CachedScore, true, _ScoreOneIn} -> % If caches are used roll the score towards the current score % Expectation is that this will reduce instances of individual % files being compacted when a run is missed due to cached % scores being used in surrounding journals NewScore = - check_single_file(JournalP, - ScoringState#scoring_state.filter_fun, - ScoringState#scoring_state.filter_server, - ScoringState#scoring_state.max_sqn, - ?SAMPLE_SIZE, - ?BATCH_SIZE, - State#state.reload_strategy), + check_single_file( + JournalP, + ScoringState#scoring_state.filter_fun, + ScoringState#scoring_state.filter_server, + ScoringState#scoring_state.max_sqn, + ?SAMPLE_SIZE, + ?BATCH_SIZE, + State#state.reload_strategy + ), (NewScore + CachedScore) / 2; {CachedScore, false, _ScoreOneIn} -> CachedScore end, ok = leveled_cdb:cdb_putcachedscore(JournalP, CpctPerc), Candidate = - #candidate{low_sqn = LowSQN, - filename = FN, - journal = JournalP, - compaction_perc = CpctPerc}, + #candidate{ + low_sqn = LowSQN, + filename = FN, + journal = JournalP, + compaction_perc = CpctPerc + }, ok = clerk_scorefilelist(self(), Tail), {noreply, State#state{scored_files = [Candidate|Candidates]}}; -handle_cast(scoring_complete, State) -> +handle_cast( + scoring_complete, State = #state{scoring_state = ScoringState}) + when ?IS_DEF(ScoringState) -> MaxRunLength = State#state.max_run_length, CDBopts = State#state.cdb_options, Candidates = lists:reverse(State#state.scored_files), - ScoringState = State#state.scoring_state, FilterFun = ScoringState#scoring_state.filter_fun, FilterServer = ScoringState#scoring_state.filter_server, MaxSQN = ScoringState#scoring_state.max_sqn, @@ -379,35 +396,45 @@ handle_cast(scoring_complete, State) -> true -> BestRun1 = sort_run(BestRun0), print_compaction_run(BestRun1, ScoreParams), - ManifestSlice = compact_files(BestRun1, - CDBopts, - FilterFun, - FilterServer, - MaxSQN, - State#state.reload_strategy, - State#state.compression_method), - FilesToDelete = lists:map(fun(C) -> - {C#candidate.low_sqn, - C#candidate.filename, - C#candidate.journal, - undefined} - end, - BestRun1), + ManifestSlice = + compact_files( + BestRun1, + CDBopts, + FilterFun, + FilterServer, + MaxSQN, + State#state.reload_strategy, + State#state.compression_method + ), + FilesToDelete = + lists:map( + fun(C) -> + { + C#candidate.low_sqn, + C#candidate.filename, + C#candidate.journal, + undefined + } + end, + BestRun1 + ), leveled_log:log(ic002, [length(FilesToDelete)]), ok = CloseFun(FilterServer), - ok = leveled_inker:ink_clerkcomplete(State#state.inker, - ManifestSlice, - FilesToDelete); + ok = + leveled_inker:ink_clerkcomplete( + State#state.inker, ManifestSlice, FilesToDelete); false -> ok = CloseFun(FilterServer), ok = leveled_inker:ink_clerkcomplete(State#state.inker, [], []) end, {noreply, State#state{scoring_state = undefined}, hibernate}; -handle_cast({trim, PersistedSQN, ManifestAsList}, State) -> +handle_cast( + {trim, PersistedSQN, ManifestAsList}, State = #state{inker = Ink}) + when ?IS_DEF(Ink) -> FilesToDelete = leveled_imanifest:find_persistedentries(PersistedSQN, ManifestAsList), leveled_log:log(ic007, []), - ok = leveled_inker:ink_clerkcomplete(State#state.inker, [], FilesToDelete), + ok = leveled_inker:ink_clerkcomplete(Ink, [], FilesToDelete), {noreply, State}; handle_cast({hashtable_calc, HashTree, StartPos, CDBpid}, State) -> {IndexList, HashTreeBin} = leveled_cdb:hashtable_calc(HashTree, StartPos), @@ -445,9 +472,9 @@ code_change(_OldVsn, State, _Extra) -> %%% External functions %%%============================================================================ --spec schedule_compaction(list(integer()), - integer(), - {integer(), integer(), integer()}) -> integer(). +-spec schedule_compaction( + list(integer()), integer(), {integer(), integer(), integer()}) -> + integer(). %% @doc %% Schedule the next compaction event for this store. Chooses a random %% interval, and then a random start time within the first third @@ -483,11 +510,11 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> % today. RandSelect = fun(_X) -> - {lists:nth(leveled_rand:uniform(TotalHours), CompactionHours), - leveled_rand:uniform(?INTERVALS_PER_HOUR)} + {lists:nth(rand:uniform(TotalHours), CompactionHours), + rand:uniform(?INTERVALS_PER_HOUR)} end, - RandIntervals = lists:sort(lists:map(RandSelect, - lists:seq(1, RunsPerDay))), + RandIntervals = + lists:sort(lists:map(RandSelect, lists:seq(1, RunsPerDay))), % Pick the next interval from the list. The intervals before current time % are considered as intervals tomorrow, so will only be next if there are @@ -508,11 +535,11 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> % Calculate the offset in seconds to this next interval NextS0 = NextI * (IntervalLength * 60) - - leveled_rand:uniform(IntervalLength * 60), + - rand:uniform(IntervalLength * 60), NextM = NextS0 div 60, NextS = NextS0 rem 60, - TimeDiff = calendar:time_difference(LocalTime, - {NextDate, {NextH, NextM, NextS}}), + TimeDiff = + calendar:time_difference(LocalTime, {NextDate, {NextH, NextM, NextS}}), {Days, {Hours, Mins, Secs}} = TimeDiff, Days * 86400 + Hours * 3600 + Mins * 60 + Secs. @@ -521,13 +548,14 @@ schedule_compaction(CompactionHours, RunsPerDay, CurrentTS) -> %%% Internal functions %%%============================================================================ --spec check_single_file(pid(), - leveled_inker:filterfun(), - leveled_inker:filterserver(), - leveled_codec:sqn(), - non_neg_integer(), non_neg_integer(), - leveled_codec:compaction_strategy()) -> - float(). +-spec check_single_file( + pid(), + leveled_inker:filterfun(), + leveled_inker:filterserver(), + leveled_codec:sqn(), + non_neg_integer(), non_neg_integer(), + leveled_codec:compaction_strategy()) -> + float(). %% @doc %% Get a score for a single CDB file in the journal. This will pull out a bunch %% of keys and sizes at random in an efficient way (by scanning the hashtable @@ -624,8 +652,8 @@ fetch_inbatches(PositionList, BatchSize, CDB, CheckedList) -> fetch_inbatches(Tail, BatchSize, CDB, CheckedList ++ KL_List). --spec assess_candidates(list(candidate()), score_parameters()) - -> {list(candidate()), float()}. +-spec assess_candidates( + list(candidate()), score_parameters()) -> {list(candidate()), float()}. %% @doc %% For each run length we need to assess all the possible runs of candidates, %% to determine which is the best score - to be put forward as the best @@ -704,10 +732,12 @@ score_run(Run, {MaxRunLength, MR_CT, SF_CT}) -> (MR_CT - SF_CT) / (MaxRunSize - 1) end, Target = SF_CT + TargetIncr * (length(Run) - 1), - RunTotal = lists:foldl(fun(Cand, Acc) -> - Acc + Cand#candidate.compaction_perc end, - 0.0, - Run), + RunTotal = + lists:foldl( + fun(Cand, Acc) -> Acc + Cand#candidate.compaction_perc end, + 0.0, + Run + ), Target - RunTotal / length(Run). @@ -750,26 +780,29 @@ compact_files([Batch|T], CDBopts, ActiveJournal0, FilterFun, FilterServer, MaxSQN, RStrategy, PressMethod, ManSlice0) -> {SrcJournal, PositionList} = Batch, - KVCs0 = leveled_cdb:cdb_directfetch(SrcJournal, - PositionList, - key_value_check), - KVCs1 = filter_output(KVCs0, - FilterFun, - FilterServer, - MaxSQN, - RStrategy), - {ActiveJournal1, ManSlice1} = write_values(KVCs1, - CDBopts, - ActiveJournal0, - ManSlice0, - PressMethod), + KVCs0 = + leveled_cdb:cdb_directfetch(SrcJournal, PositionList, key_value_check), + KVCs1 = + filter_output(KVCs0, FilterFun, FilterServer, MaxSQN, RStrategy), + {ActiveJournal1, ManSlice1} = + write_values( + KVCs1, CDBopts, ActiveJournal0, ManSlice0, PressMethod), % The inker's clerk will no longer need these (potentially large) binaries, % so force garbage collection at this point. This will mean when we roll % each CDB file there will be no remaining references to the binaries that % have been transferred and the memory can immediately be cleared garbage_collect(), - compact_files(T, CDBopts, ActiveJournal1, FilterFun, FilterServer, MaxSQN, - RStrategy, PressMethod, ManSlice1). + compact_files( + T, + CDBopts, + ActiveJournal1, + FilterFun, + FilterServer, + MaxSQN, + RStrategy, + PressMethod, + ManSlice1 + ). get_all_positions([], PositionBatches) -> PositionBatches; @@ -777,23 +810,25 @@ get_all_positions([HeadRef|RestOfBest], PositionBatches) -> SrcJournal = HeadRef#candidate.journal, Positions = leveled_cdb:cdb_getpositions(SrcJournal, all), leveled_log:log(ic008, [HeadRef#candidate.filename, length(Positions)]), - Batches = split_positions_into_batches(lists:sort(Positions), - SrcJournal, - []), + Batches = + split_positions_into_batches( + lists:sort(Positions), SrcJournal, [] + ), get_all_positions(RestOfBest, PositionBatches ++ Batches). split_positions_into_batches([], _Journal, Batches) -> Batches; split_positions_into_batches(Positions, Journal, Batches) -> - {ThisBatch, Tail} = if - length(Positions) > ?BATCH_SIZE -> - lists:split(?BATCH_SIZE, Positions); - true -> - {Positions, []} - end, - split_positions_into_batches(Tail, - Journal, - Batches ++ [{Journal, ThisBatch}]). + {ThisBatch, Tail} = + if + length(Positions) > ?BATCH_SIZE -> + lists:split(?BATCH_SIZE, Positions); + true -> + {Positions, []} + end, + split_positions_into_batches( + Tail, Journal, Batches ++ [{Journal, ThisBatch}] + ). %% @doc @@ -918,7 +953,7 @@ clear_waste(State) -> N = calendar:datetime_to_gregorian_seconds(calendar:local_time()), DeleteJournalFun = fun(DelJ) -> - LMD = filelib:last_modified(WP ++ DelJ), + LMD = {_,_} = filelib:last_modified(WP ++ DelJ), case N - calendar:datetime_to_gregorian_seconds(LMD) of LMD_Delta when LMD_Delta >= WRP -> ok = file:delete(WP ++ DelJ), @@ -931,12 +966,10 @@ clear_waste(State) -> lists:foreach(DeleteJournalFun, ClearedJournals) end. - %%%============================================================================ %%% Test %%%============================================================================ - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -966,24 +999,36 @@ local_time_to_now(DateTime) -> {Seconds div 1000000, Seconds rem 1000000, 0}. simple_score_test() -> - Run1 = [#candidate{compaction_perc = 75.0}, - #candidate{compaction_perc = 75.0}, - #candidate{compaction_perc = 76.0}, - #candidate{compaction_perc = 70.0}], + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Run1 = [DummyC#candidate{compaction_perc = 75.0}, + DummyC#candidate{compaction_perc = 75.0}, + DummyC#candidate{compaction_perc = 76.0}, + DummyC#candidate{compaction_perc = 70.0}], ?assertMatch(-4.0, score_run(Run1, {4, 70.0, 40.0})), - Run2 = [#candidate{compaction_perc = 75.0}], + Run2 = [DummyC#candidate{compaction_perc = 75.0}], ?assertMatch(-35.0, score_run(Run2, {4, 70.0, 40.0})), ?assertEqual(0.0, score_run([], {4, 40.0, 70.0})), - Run3 = [#candidate{compaction_perc = 100.0}], + Run3 = [DummyC#candidate{compaction_perc = 100.0}], ?assertMatch(-60.0, score_run(Run3, {4, 70.0, 40.0})). file_gc_test() -> - State = #state{waste_path="test/test_area/waste/", - waste_retention_period=1}, + State = + #state{ + waste_path="test/test_area/waste/", waste_retention_period = 1 + }, ok = filelib:ensure_dir(State#state.waste_path), - file:write_file(State#state.waste_path ++ "1.cdb", term_to_binary("Hello")), + file:write_file( + filename:join(State#state.waste_path, "1.cdb"), + term_to_binary("Hello") + ), timer:sleep(1100), - file:write_file(State#state.waste_path ++ "2.cdb", term_to_binary("Hello")), + file:write_file( + filename:join(State#state.waste_path, "2.cdb"), + term_to_binary("Hello") + ), clear_waste(State), {ok, ClearedJournals} = file:list_dir(State#state.waste_path), ?assertMatch(["2.cdb"], ClearedJournals), @@ -1004,27 +1049,47 @@ find_bestrun_test() -> %% -define(MAXRUNLENGTH_COMPACTION_TARGET, 60.0). %% Tested first with blocks significant as no back-tracking Params = {4, 60.0, 40.0}, - Block1 = [#candidate{compaction_perc = 55.0, filename = "a"}, - #candidate{compaction_perc = 65.0, filename = "b"}, - #candidate{compaction_perc = 42.0, filename = "c"}, - #candidate{compaction_perc = 50.0, filename = "d"}], - Block2 = [#candidate{compaction_perc = 38.0, filename = "e"}, - #candidate{compaction_perc = 75.0, filename = "f"}, - #candidate{compaction_perc = 75.0, filename = "g"}, - #candidate{compaction_perc = 45.0, filename = "h"}], - Block3 = [#candidate{compaction_perc = 70.0, filename = "i"}, - #candidate{compaction_perc = 100.0, filename = "j"}, - #candidate{compaction_perc = 100.0, filename = "k"}, - #candidate{compaction_perc = 100.0, filename = "l"}], - Block4 = [#candidate{compaction_perc = 55.0, filename = "m"}, - #candidate{compaction_perc = 56.0, filename = "n"}, - #candidate{compaction_perc = 57.0, filename = "o"}, - #candidate{compaction_perc = 40.0, filename = "p"}], - Block5 = [#candidate{compaction_perc = 60.0, filename = "q"}, - #candidate{compaction_perc = 60.0, filename = "r"}], + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Block1 = + [ + DummyC#candidate{compaction_perc = 55.0, filename = "a"}, + DummyC#candidate{compaction_perc = 65.0, filename = "b"}, + DummyC#candidate{compaction_perc = 42.0, filename = "c"}, + DummyC#candidate{compaction_perc = 50.0, filename = "d"} + ], + Block2 = + [ + DummyC#candidate{compaction_perc = 38.0, filename = "e"}, + DummyC#candidate{compaction_perc = 75.0, filename = "f"}, + DummyC#candidate{compaction_perc = 75.0, filename = "g"}, + DummyC#candidate{compaction_perc = 45.0, filename = "h"} + ], + Block3 = + [ + DummyC#candidate{compaction_perc = 70.0, filename = "i"}, + DummyC#candidate{compaction_perc = 100.0, filename = "j"}, + DummyC#candidate{compaction_perc = 100.0, filename = "k"}, + DummyC#candidate{compaction_perc = 100.0, filename = "l"} + ], + Block4 = + [ + DummyC#candidate{compaction_perc = 55.0, filename = "m"}, + DummyC#candidate{compaction_perc = 56.0, filename = "n"}, + DummyC#candidate{compaction_perc = 57.0, filename = "o"}, + DummyC#candidate{compaction_perc = 40.0, filename = "p"} + ], + Block5 = + [ + DummyC#candidate{compaction_perc = 60.0, filename = "q"}, + DummyC#candidate{compaction_perc = 60.0, filename = "r"} + ], CList0 = Block1 ++ Block2 ++ Block3 ++ Block4 ++ Block5, ?assertMatch(["b", "c", "d", "e"], check_bestrun(CList0, Params)), - CList1 = CList0 ++ [#candidate{compaction_perc = 20.0, filename="s"}], + CList1 = + CList0 ++ [DummyC#candidate{compaction_perc = 20.0, filename="s"}], ?assertMatch(["s"], check_bestrun(CList1, Params)), CList2 = Block4 ++ Block3 ++ Block2 ++ Block1 ++ Block5, ?assertMatch(["h", "a", "b", "c"], check_bestrun(CList2, Params)), @@ -1219,12 +1284,18 @@ compact_empty_file_test() -> ok = leveled_cdb:cdb_destroy(CDB2). compare_candidate_test() -> - Candidate1 = #candidate{low_sqn=1}, - Candidate2 = #candidate{low_sqn=2}, - Candidate3 = #candidate{low_sqn=3}, - Candidate4 = #candidate{low_sqn=4}, - ?assertMatch([Candidate1, Candidate2, Candidate3, Candidate4], - sort_run([Candidate3, Candidate2, Candidate4, Candidate1])). + DummyC = + #candidate{ + low_sqn = 1, filename="dummy", journal=self(), compaction_perc = 0 + }, + Candidate1 = DummyC#candidate{low_sqn=1}, + Candidate2 = DummyC#candidate{low_sqn=2}, + Candidate3 = DummyC#candidate{low_sqn=3}, + Candidate4 = DummyC#candidate{low_sqn=4}, + ?assertMatch( + [Candidate1, Candidate2, Candidate3, Candidate4], + sort_run([Candidate3, Candidate2, Candidate4, Candidate1]) + ). compact_singlefile_totwosmallfiles_test_() -> {timeout, 60, fun compact_singlefile_totwosmallfiles_testto/0}. @@ -1236,24 +1307,31 @@ compact_singlefile_totwosmallfiles_testto() -> FN1 = leveled_inker:filepath(RP, 1, new_journal), CDBoptsLarge = #cdb_options{binary_mode=true, max_size=30000000}, {ok, CDB1} = leveled_cdb:cdb_open_writer(FN1, CDBoptsLarge), - lists:foreach(fun(X) -> - LK = test_ledgerkey("Key" ++ integer_to_list(X)), - Value = leveled_rand:rand_bytes(1024), - {IK, IV} = - leveled_codec:to_inkerkv(LK, X, Value, - {[], infinity}, - native, true), - ok = leveled_cdb:cdb_put(CDB1, IK, IV) - end, - lists:seq(1, 1000)), + lists:foreach( + fun(X) -> + LK = test_ledgerkey("Key" ++ integer_to_list(X)), + Value = crypto:strong_rand_bytes(1024), + {IK, IV} = + leveled_codec:to_inkerkv(LK, X, Value, + {[], infinity}, + native, true), + ok = leveled_cdb:cdb_put(CDB1, IK, IV) + end, + lists:seq(1, 1000) + ), {ok, NewName} = leveled_cdb:cdb_complete(CDB1), {ok, CDBr} = leveled_cdb:cdb_open_reader(NewName), CDBoptsSmall = #cdb_options{binary_mode=true, max_size=400000, file_path=CP}, - BestRun1 = [#candidate{low_sqn=1, - filename=leveled_cdb:cdb_filename(CDBr), - journal=CDBr, - compaction_perc=50.0}], + BestRun1 = + [ + #candidate{ + low_sqn=1, + filename=leveled_cdb:cdb_filename(CDBr), + journal=CDBr, + compaction_perc=50.0 + } + ], FakeFilterFun = fun(_FS, _LK, SQN) -> case SQN rem 2 of @@ -1262,19 +1340,24 @@ compact_singlefile_totwosmallfiles_testto() -> end end, - ManifestSlice = compact_files(BestRun1, - CDBoptsSmall, - FakeFilterFun, - null, - 900, - [{?STD_TAG, recovr}], - native), + ManifestSlice = + compact_files( + BestRun1, + CDBoptsSmall, + FakeFilterFun, + null, + 900, + [{?STD_TAG, recovr}], + native + ), ?assertMatch(2, length(ManifestSlice)), - lists:foreach(fun({_SQN, _FN, CDB, _LK}) -> - ok = leveled_cdb:cdb_deletepending(CDB), - ok = leveled_cdb:cdb_destroy(CDB) - end, - ManifestSlice), + lists:foreach( + fun({_SQN, _FN, CDB, _LK}) -> + ok = leveled_cdb:cdb_deletepending(CDB), + ok = leveled_cdb:cdb_destroy(CDB) + end, + ManifestSlice + ), ok = leveled_cdb:cdb_deletepending(CDBr), ok = leveled_cdb:cdb_destroy(CDBr). @@ -1304,11 +1387,13 @@ size_score_test() -> end end, Score = - size_comparison_score(KeySizeList, - FilterFun, - CurrentList, - MaxSQN, - leveled_codec:inker_reload_strategy([])), + size_comparison_score( + KeySizeList, + FilterFun, + CurrentList, + MaxSQN, + leveled_codec:inker_reload_strategy([]) + ), io:format("Score ~w", [Score]), ?assertMatch(true, Score > 69.0), ?assertMatch(true, Score < 70.0). diff --git a/src/leveled_imanifest.erl b/src/leveled_imanifest.erl index 480c1974..919c240c 100644 --- a/src/leveled_imanifest.erl +++ b/src/leveled_imanifest.erl @@ -28,9 +28,7 @@ -type manifest() :: list({integer(), list()}). %% The manifest is divided into blocks by sequence number, with each block %% being a list of manifest entries for that SQN range. --type manifest_entry() :: {integer(), string(), pid()|string(), any()}. -%% The Entry should have a pid() as the third element, but a string() may be -%% used in unit tests +-type manifest_entry() :: {integer(), string(), pid(), any()}. -export_type([manifest/0, manifest_entry/0]). @@ -75,8 +73,8 @@ add_entry(Manifest, Entry, ToEnd) -> from_list(Man1) end. --spec append_lastkey(manifest(), pid(), leveled_codec:journal_key()) - -> manifest(). +-spec append_lastkey( + manifest(), pid(), leveled_codec:journal_key()) -> manifest(). %% @doc %% On discovery of the last key in the last journal entry, the manifest can %% be updated through this function to have the last key @@ -100,7 +98,7 @@ remove_entry(Manifest, Entry) -> Man0 = lists:keydelete(SQN, 1, to_list(Manifest)), from_list(Man0). --spec find_entry(integer(), manifest()) -> pid()|string(). +-spec find_entry(integer(), manifest()) -> pid(). %% @doc %% Given a SQN find the relevant manifest_entry, returning just the pid() of %% the journal file (which may be a string() in unit tests) @@ -257,18 +255,31 @@ build_testmanifest_aslist() -> ManifestMapFun = fun(N) -> NStr = integer_to_list(N), - {max(1, N * 1000), "FN" ++ NStr, "pid" ++ NStr, "LK" ++ NStr} + { + max(1, N * 1000), + "FN" ++ NStr, + set_pid(N), + "LK" ++ NStr + } end, lists:map(ManifestMapFun, lists:reverse(lists:seq(0, 50))). +set_pid(N) -> + lists:flatten(io_lib:format("<0.1~2..0w.0>", [N])). + test_testmanifest(Man0) -> - ?assertMatch("pid0", find_entry(1, Man0)), - ?assertMatch("pid0", find_entry(2, Man0)), - ?assertMatch("pid1", find_entry(1001, Man0)), - ?assertMatch("pid20", find_entry(20000, Man0)), - ?assertMatch("pid20", find_entry(20001, Man0)), - ?assertMatch("pid20", find_entry(20999, Man0)), - ?assertMatch("pid50", find_entry(99999, Man0)). + P0 = set_pid(0), + P1 = set_pid(1), + P20 = set_pid(20), + P50 = set_pid(50), + + ?assertMatch(P0, find_entry(1, Man0)), + ?assertMatch(P0, find_entry(2, Man0)), + ?assertMatch(P1, find_entry(1001, Man0)), + ?assertMatch(P20, find_entry(20000, Man0)), + ?assertMatch(P20, find_entry(20001, Man0)), + ?assertMatch(P20, find_entry(20999, Man0)), + ?assertMatch(P50, find_entry(99999, Man0)). buildfromlist_test() -> ManL = build_testmanifest_aslist(), @@ -303,7 +314,7 @@ buildrandomfashion_test() -> ManL0 = build_testmanifest_aslist(), RandMapFun = fun(X) -> - {leveled_rand:uniform(), X} + {rand:uniform(), X} end, ManL1 = lists:map(RandMapFun, ManL0), ManL2 = lists:sort(ManL1), @@ -317,7 +328,7 @@ buildrandomfashion_test() -> test_testmanifest(Man0), ?assertMatch(ManL0, to_list(Man0)), - RandomEntry = lists:nth(leveled_rand:uniform(50), ManL0), + RandomEntry = lists:nth(rand:uniform(50), ManL0), Man1 = remove_entry(Man0, RandomEntry), Man2 = add_entry(Man1, RandomEntry, false), diff --git a/src/leveled_inker.erl b/src/leveled_inker.erl index 6ad5ced9..9721267a 100644 --- a/src/leveled_inker.erl +++ b/src/leveled_inker.erl @@ -147,7 +147,7 @@ -record(state, {manifest = [] :: list(), manifest_sqn = 0 :: integer(), - journal_sqn = 0 :: integer(), + journal_sqn = 0 :: non_neg_integer(), active_journaldb :: pid() | undefined, pending_removals = [] :: list(), registered_snapshots = [] :: list(registered_snapshot()), @@ -159,7 +159,9 @@ is_snapshot = false :: boolean(), compression_method = native :: lz4|native|none, compress_on_receipt = false :: boolean(), - snap_timeout :: pos_integer() | undefined, % in seconds + snap_timeout = 0 :: non_neg_integer(), + % in seconds, 0 for snapshots + % (only relevant for primary Inker) source_inker :: pid() | undefined, shutdown_loops = ?SHUTDOWN_LOOPS :: non_neg_integer()}). @@ -196,20 +198,26 @@ %% The inker will need to know what the reload strategy is, to inform the %% clerk about the rules to enforce during compaction. ink_start(InkerOpts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), InkerOpts], []). + {ok, Inker} = + gen_server:start_link( + ?MODULE, [leveled_log:get_opts(), InkerOpts], []), + {ok, Inker}. -spec ink_snapstart(inker_options()) -> {ok, pid()}. %% @doc %% Don't link on startup as snapshot ink_snapstart(InkerOpts) -> - gen_server:start(?MODULE, [leveled_log:get_opts(), InkerOpts], []). + {ok, Inker} = + gen_server:start( + ?MODULE, [leveled_log:get_opts(), InkerOpts], []), + {ok, Inker}. -spec ink_put(pid(), leveled_codec:ledger_key(), any(), leveled_codec:journal_keychanges(), boolean()) -> - {ok, integer(), integer()}. + {ok, non_neg_integer(), pos_integer()}. %% @doc %% PUT an object into the journal, returning the sequence number for the PUT %% as well as the size of the object (information required by the ledger). @@ -504,14 +512,13 @@ ink_getclerkpid(Pid) -> init([LogOpts, InkerOpts]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), case {InkerOpts#inker_options.root_path, - InkerOpts#inker_options.start_snapshot} of - {undefined, true} -> + InkerOpts#inker_options.start_snapshot, + InkerOpts#inker_options.source_inker} of + {undefined, true, SrcInker} when ?IS_DEF(SrcInker) -> %% monitor the bookie, and close the snapshot when bookie %% exits BookieMonitor = erlang:monitor(process, InkerOpts#inker_options.bookies_pid), - SrcInker = InkerOpts#inker_options.source_inker, {Manifest, ActiveJournalDB, JournalSQN} = ink_registersnapshot(SrcInker, self()), @@ -522,7 +529,7 @@ init([LogOpts, InkerOpts]) -> bookie_monref = BookieMonitor, is_snapshot = true}}; %% Need to do something about timeout - {_RootPath, false} -> + {_RootPath, false, _SrcInker} -> start_from_file(InkerOpts) end. @@ -557,10 +564,12 @@ handle_call({fold, Manifest = lists:reverse(leveled_imanifest:to_list(State#state.manifest)), Folder = fun() -> - fold_from_sequence(StartSQN, - {FilterFun, InitAccFun, FoldFun}, - Acc, - Manifest) + fold_from_sequence( + StartSQN, + {FilterFun, InitAccFun, FoldFun}, + Acc, + Manifest + ) end, case By of as_ink -> @@ -583,25 +592,21 @@ handle_call(get_manifest, _From, State) -> handle_call(print_manifest, _From, State) -> leveled_imanifest:printer(State#state.manifest), {reply, ok, State}; -handle_call({compact, - Checker, - InitiateFun, - CloseFun, - FilterFun}, - _From, State=#state{is_snapshot=Snap}) when Snap == false -> +handle_call( + {compact, Checker, InitiateFun, CloseFun, FilterFun}, + _From, + State=#state{is_snapshot=Snap}) + when Snap == false -> Clerk = State#state.clerk, Manifest = leveled_imanifest:to_list(State#state.manifest), - leveled_iclerk:clerk_compact(State#state.clerk, - Checker, - InitiateFun, - CloseFun, - FilterFun, - Manifest), + leveled_iclerk:clerk_compact( + Clerk, Checker, InitiateFun, CloseFun, FilterFun, Manifest), {reply, {ok, Clerk}, State#state{compaction_pending=true}}; handle_call(compaction_pending, _From, State) -> {reply, State#state.compaction_pending, State}; -handle_call({trim, PersistedSQN}, _From, State=#state{is_snapshot=Snap}) - when Snap == false -> +handle_call( + {trim, PersistedSQN}, _From, State=#state{is_snapshot=Snap}) + when Snap == false -> Manifest = leveled_imanifest:to_list(State#state.manifest), ok = leveled_iclerk:clerk_trim(State#state.clerk, PersistedSQN, Manifest), {reply, ok, State}; @@ -625,8 +630,9 @@ handle_call(roll, _From, State=#state{is_snapshot=Snap}) when Snap == false -> manifest_sqn = NewManSQN, active_journaldb = NewJournalP}} end; -handle_call({backup, BackupPath}, _from, State) - when State#state.is_snapshot == true -> +handle_call( + {backup, BackupPath}, _from, State) + when State#state.is_snapshot == true -> SW = os:timestamp(), BackupJFP = filepath(filename:join(BackupPath, ?JOURNAL_FP), journal_dir), ok = filelib:ensure_dir(BackupJFP), @@ -665,7 +671,7 @@ handle_call({backup, BackupPath}, _from, State) leveled_log:log(i0022, [RFN]), RemoveFile = filename:join(BackupJFP, RFN), case filelib:is_file(RemoveFile) - and not filelib:is_dir(RemoveFile) of + andalso not filelib:is_dir(RemoveFile) of true -> ok = file:delete(RemoveFile); false -> @@ -699,12 +705,13 @@ handle_call(get_clerkpid, _From, State) -> handle_call(close, _From, State=#state{is_snapshot=Snap}) when Snap == true -> ok = ink_releasesnapshot(State#state.source_inker, self()), {stop, normal, ok, State}; -handle_call(ShutdownType, From, State) - when ShutdownType == close; ShutdownType == doom -> +handle_call( + ShutdownType, From, State = #state{clerk = Clerk}) + when ?IS_DEF(Clerk) -> case ShutdownType of doom -> leveled_log:log(i0018, []); - _ -> + close -> ok end, leveled_log:log(i0005, [ShutdownType]), @@ -714,9 +721,10 @@ handle_call(ShutdownType, From, State) gen_server:cast(self(), {maybe_defer_shutdown, ShutdownType, From}), {noreply, State}. - -handle_cast({clerk_complete, ManifestSnippet, FilesToDelete}, State) -> - CDBOpts = State#state.cdb_options, +handle_cast( + {clerk_complete, ManifestSnippet, FilesToDelete}, + State = #state{cdb_options = CDBOpts}) + when ?IS_DEF(CDBOpts) -> DropFun = fun(E, Acc) -> leveled_imanifest:remove_entry(Acc, E) @@ -854,8 +862,10 @@ handle_cast({complete_shutdown, ShutdownType, From}, State) -> {stop, normal, State}. %% handle the bookie stopping and stop this snapshot -handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info}, - State=#state{bookie_monref = BookieMonRef}) -> +handle_info( + {'DOWN', BookieMonRef, process, _BookiePid, _Info}, + State=#state{bookie_monref = BookieMonRef, source_inker = SrcInker}) + when ?IS_DEF(SrcInker) -> %% Monitor only registered on snapshots ok = ink_releasesnapshot(State#state.source_inker, self()), {stop, normal, State}; @@ -879,7 +889,10 @@ code_change(_OldVsn, State, _Extra) -> -spec start_from_file(inker_options()) -> {ok, ink_state()}. %% @doc %% Start an Inker from the state on disk (i.e. not a snapshot). -start_from_file(InkOpts) -> +start_from_file( + InkOpts = + #inker_options{root_path = RootPath, snaptimeout_long = SnapTimeout}) + when ?IS_DEF(RootPath), ?IS_DEF(SnapTimeout) -> % Setting the correct CDB options is important when starting the inker, in % particular for waste retention which is determined by the CDB options % with which the file was last opened @@ -926,9 +939,8 @@ start_from_file(InkOpts) -> {Manifest, ManifestSQN, JournalSQN, - ActiveJournal} = build_manifest(ManifestFilenames, - RootPath, - CDBopts), + ActiveJournal} = + build_manifest(ManifestFilenames, RootPath, CDBopts), {ok, #state{manifest = Manifest, manifest_sqn = ManifestSQN, journal_sqn = JournalSQN, @@ -971,61 +983,74 @@ get_cdbopts(InkOpts)-> CDBopts#cdb_options{waste_path = WasteFP}. --spec put_object(leveled_codec:ledger_key(), - any(), - leveled_codec:journal_keychanges(), - boolean(), - ink_state()) - -> {ok|rolling, ink_state(), integer()}. +-spec put_object( + leveled_codec:primary_key(), + any(), + leveled_codec:journal_keychanges(), + boolean(), + ink_state()) + -> {ok|rolling, ink_state(), integer()}. %% @doc %% Add the object to the current journal if it fits. If it doesn't fit, a new %% journal must be started, and the old journal is set to "roll" into a read %% only Journal. %% The reply contains the byte_size of the object, using the size calculated %% to store the object. -put_object(LedgerKey, Object, KeyChanges, Sync, State) -> +put_object( + LedgerKey, + Object, + KeyChanges, + Sync, + State = + #state{ + active_journaldb = ActiveJournal, + cdb_options = CDBOpts, + root_path = RP + }) + when ?IS_DEF(ActiveJournal), ?IS_DEF(CDBOpts), ?IS_DEF(RP) -> NewSQN = State#state.journal_sqn + 1, - ActiveJournal = State#state.active_journaldb, {JournalKey, JournalBin} = - leveled_codec:to_inkerkv(LedgerKey, - NewSQN, - Object, - KeyChanges, - State#state.compression_method, - State#state.compress_on_receipt), - case leveled_cdb:cdb_put(ActiveJournal, - JournalKey, - JournalBin, - Sync) of + leveled_codec:to_inkerkv( + LedgerKey, + NewSQN, + Object, + KeyChanges, + State#state.compression_method, + State#state.compress_on_receipt + ), + PutR = leveled_cdb:cdb_put(ActiveJournal, JournalKey, JournalBin, Sync), + case PutR of ok -> - {ok, - State#state{journal_sqn=NewSQN}, - byte_size(JournalBin)}; + {ok, State#state{journal_sqn=NewSQN}, byte_size(JournalBin)}; roll -> SWroll = os:timestamp(), {NewJournalP, Manifest1, NewManSQN} = - roll_active(ActiveJournal, - State#state.manifest, - NewSQN, - State#state.cdb_options, - State#state.root_path, - State#state.manifest_sqn), + roll_active( + ActiveJournal, + State#state.manifest, + NewSQN, + State#state.cdb_options, + State#state.root_path, + State#state.manifest_sqn + ), leveled_log:log_timer(i0008, [], SWroll), - ok = leveled_cdb:cdb_put(NewJournalP, - JournalKey, - JournalBin), + ok = + leveled_cdb:cdb_put( + NewJournalP, JournalKey, JournalBin), {rolling, - State#state{journal_sqn=NewSQN, - manifest=Manifest1, - manifest_sqn = NewManSQN, - active_journaldb=NewJournalP}, + State#state{ + journal_sqn=NewSQN, + manifest=Manifest1, + manifest_sqn = NewManSQN, + active_journaldb=NewJournalP}, byte_size(JournalBin)} end. --spec get_object(leveled_codec:ledger_key(), - integer(), - leveled_imanifest:manifest()) -> any(). +-spec get_object( + leveled_codec:ledger_key(), + integer(), + leveled_imanifest:manifest()) -> any(). %% @doc %% Find the SQN in the manifest and then fetch the object from the Journal, %% in the manifest. If the fetch is in response to a user GET request then @@ -1041,28 +1066,36 @@ get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> leveled_codec:from_inkerkv(Obj, ToIgnoreKeyChanges). --spec roll_active(pid(), leveled_imanifest:manifest(), - integer(), #cdb_options{}, string(), integer()) -> - {pid(), leveled_imanifest:manifest(), integer()}. +-spec roll_active( + pid(), + leveled_imanifest:manifest(), + integer(), + #cdb_options{}, + string(), + integer()) -> {pid(), leveled_imanifest:manifest(), integer()}. %% @doc %% Roll the active journal, and start a new active journal, updating the %% manifest roll_active(ActiveJournal, Manifest, NewSQN, CDBopts, RootPath, ManifestSQN) -> - LastKey = leveled_cdb:cdb_lastkey(ActiveJournal), - ok = leveled_cdb:cdb_roll(ActiveJournal), - Manifest0 = - leveled_imanifest:append_lastkey(Manifest, ActiveJournal, LastKey), - ManEntry = - start_new_activejournal(NewSQN, RootPath, CDBopts), - {_, _, NewJournalP, _} = ManEntry, - Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), - ok = leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), - - {NewJournalP, Manifest1, ManifestSQN + 1}. + case leveled_cdb:cdb_lastkey(ActiveJournal) of + LastKey when LastKey =/= empty -> + ok = leveled_cdb:cdb_roll(ActiveJournal), + Manifest0 = + leveled_imanifest:append_lastkey(Manifest, ActiveJournal, LastKey), + ManEntry = + start_new_activejournal(NewSQN, RootPath, CDBopts), + {_, _, NewJournalP, _} = ManEntry, + Manifest1 = leveled_imanifest:add_entry(Manifest0, ManEntry, true), + ok = + leveled_imanifest:writer(Manifest1, ManifestSQN + 1, RootPath), + + {NewJournalP, Manifest1, ManifestSQN + 1} + end. --spec key_check(leveled_codec:ledger_key(), - integer(), - leveled_imanifest:manifest()) -> missing|probably. +-spec key_check( + leveled_codec:primary_key(), + integer(), + leveled_imanifest:manifest()) -> missing|probably. %% @doc %% Checks for the presence of the key at that SQN withing the journal, %% avoiding the cost of actually reading the object from disk. @@ -1081,40 +1114,36 @@ key_check(LedgerKey, SQN, Manifest) -> %% Selects the correct manifest to open, and then starts a process for each %% file in the manifest, storing the PID for that process within the manifest. %% Opens an active journal if one is not present. -build_manifest(ManifestFilenames, - RootPath, - CDBopts) -> +build_manifest(ManifestFilenames, RootPath, CDBopts) -> % Find the manifest with a highest Manifest sequence number % Open it and read it to get the current Confirmed Manifest ManifestRegex = "(?[0-9]+)\\." ++ leveled_imanifest:complete_filex(), - ValidManSQNs = sequencenumbers_fromfilenames(ManifestFilenames, - ManifestRegex, - 'MSQN'), - {Manifest, - ManifestSQN} = case length(ValidManSQNs) of - 0 -> - {[], 1}; - _ -> - PersistedManSQN = lists:max(ValidManSQNs), - M1 = leveled_imanifest:reader(PersistedManSQN, - RootPath), - {M1, PersistedManSQN} - end, + ValidManSQNs = + sequencenumbers_fromfilenames( + ManifestFilenames, ManifestRegex, 'MSQN'), + {Manifest, ManifestSQN} = + case length(ValidManSQNs) of + 0 -> + {[], 1}; + _ -> + PersistedManSQN = lists:max(ValidManSQNs), + M1 = leveled_imanifest:reader(PersistedManSQN, RootPath), + {M1, PersistedManSQN} + end, % Open the manifest files, completing if necessary and ensure there is % a valid active journal at the head of the manifest OpenManifest = open_all_manifest(Manifest, RootPath, CDBopts), - {ActiveLowSQN, - _FN, - ActiveJournal, - _LK} = leveled_imanifest:head_entry(OpenManifest), - JournalSQN = case leveled_cdb:cdb_lastkey(ActiveJournal) of - empty -> - ActiveLowSQN; - {JSQN, _Type, _LastKey} -> - JSQN - end, + {ActiveLowSQN, _FN, ActiveJournal, _LK} = + leveled_imanifest:head_entry(OpenManifest), + JournalSQN = + case leveled_cdb:cdb_lastkey(ActiveJournal) of + empty -> + ActiveLowSQN; + {JSQN, _Type, _LastKey} -> + JSQN + end, % Update the manifest if it has been changed by the process of loading % the manifest (must also increment the manifest SQN). @@ -1146,8 +1175,9 @@ close_allmanifest([H|ManifestT]) -> close_allmanifest(ManifestT). --spec open_all_manifest(leveled_imanifest:manifest(), list(), #cdb_options{}) - -> leveled_imanifest:manifest(). +-spec open_all_manifest( + leveled_imanifest:manifest(), list(), #cdb_options{}) + -> leveled_imanifest:manifest(). %% @doc %% Open all the files in the manifets, and updating the manifest with the PIDs %% of the opened files @@ -1185,24 +1215,21 @@ open_all_manifest(Man0, RootPath, CDBOpts) -> true -> leveled_log:log(i0012, [HeadFN]), {ok, HeadR} = leveled_cdb:cdb_open_reader(CompleteHeadFN), - LastKey = leveled_cdb:cdb_lastkey(HeadR), - LastSQN = element(1, LastKey), - ManToHead = leveled_imanifest:add_entry(OpenedTail, - {HeadSQN, - HeadFN, - HeadR, - LastKey}, - true), - NewManEntry = start_new_activejournal(LastSQN + 1, - RootPath, - CDBOpts), + LastKey = {LastSQN, _, _} = leveled_cdb:cdb_lastkey(HeadR), + ManToHead = + leveled_imanifest:add_entry( + OpenedTail, + {HeadSQN, HeadFN, HeadR, LastKey}, + true + ), + NewManEntry = + start_new_activejournal(LastSQN + 1, RootPath, CDBOpts), leveled_imanifest:add_entry(ManToHead, NewManEntry, true); false -> - {ok, HeadW} = leveled_cdb:cdb_open_writer(PendingHeadFN, - CDBOpts), - leveled_imanifest:add_entry(OpenedTail, - {HeadSQN, HeadFN, HeadW, HeadLK}, - true) + {ok, HeadW} = + leveled_cdb:cdb_open_writer(PendingHeadFN, CDBOpts), + leveled_imanifest:add_entry( + OpenedTail, {HeadSQN, HeadFN, HeadW, HeadLK}, true) end. @@ -1288,17 +1315,19 @@ foldfile_between_sequence(MinSQN, MaxSQN, FoldFuns, sequencenumbers_fromfilenames(Filenames, Regex, IntName) -> - lists:foldl(fun(FN, Acc) -> - case re:run(FN, - Regex, - [{capture, [IntName], list}]) of - nomatch -> - Acc; - {match, [Int]} when is_list(Int) -> - Acc ++ [list_to_integer(Int)] - end end, - [], - Filenames). + lists:foldl( + fun(FN, Acc) -> + case re:run(FN, + Regex, + [{capture, [IntName], list}]) of + nomatch -> + Acc; + {match, [Int]} when is_list(Int) -> + Acc ++ [list_to_integer(Int)] + end + end, + [], + Filenames). filepath(RootPath, journal_dir) -> RootPath ++ "/" ++ ?FILES_FP ++ "/"; @@ -1525,7 +1554,7 @@ compact_journal_testto(WRP, ExpectedFiles) -> PK = "KeyZ" ++ integer_to_list(X), {ok, SQN, _} = ink_put(Ink1, test_ledgerkey(PK), - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), {[], infinity}, false), {SQN, test_ledgerkey(PK)} diff --git a/src/leveled_log.erl b/src/leveled_log.erl index f698201d..d35a6557 100644 --- a/src/leveled_log.erl +++ b/src/leveled_log.erl @@ -377,7 +377,7 @@ get_opts() -> } end. --spec return_settings() -> {log_level(), list(string())}. +-spec return_settings() -> {log_level(), list(atom())}. %% @doc %% Return the settings outside of the record return_settings() -> @@ -454,7 +454,7 @@ log_timer(LogRef, Subs, StartTime, SupportedLevels) -> -spec log_randomtimer(atom(), list(), erlang:timestamp(), float()) -> ok. log_randomtimer(LogReference, Subs, StartTime, RandomProb) -> - R = leveled_rand:uniform(), + R = rand:uniform(), case R < RandomProb of true -> log_timer(LogReference, Subs, StartTime); diff --git a/src/leveled_monitor.erl b/src/leveled_monitor.erl index 78b10db0..4e776f78 100644 --- a/src/leveled_monitor.erl +++ b/src/leveled_monitor.erl @@ -136,13 +136,13 @@ {leveled_pmanifest:lsm_level(), #sst_fetch_timings{}}. -type log_type() :: bookie_head|bookie_get|bookie_put|bookie_snap|pcl_fetch|sst_fetch|cdb_get. --type pcl_level() :: mem|leveled_pmanifest:lsm_level(). +-type pcl_level() :: memory|leveled_pmanifest:lsm_level(). -type sst_fetch_type() :: fetch_cache|slot_cachedblock|slot_noncachedblock|not_found. -type microsecs() :: pos_integer(). -type byte_size() :: pos_integer(). -type monitor() :: {no_monitor, 0}|{pid(), 0..100}. --type timing() :: no_timing|pos_integer(). +-type timing() :: no_timing|microsecs(). -type bookie_get_update() :: @@ -173,8 +173,10 @@ -spec monitor_start(pos_integer(), list(log_type())) -> {ok, pid()}. monitor_start(LogFreq, LogOrder) -> - gen_server:start_link( - ?MODULE, [leveled_log:get_opts(), LogFreq, LogOrder], []). + {ok, Monitor} = + gen_server:start_link( + ?MODULE, [leveled_log:get_opts(), LogFreq, LogOrder], []), + {ok, Monitor}. -spec add_stat(pid(), statistic()) -> ok. add_stat(Watcher, Statistic) -> @@ -204,7 +206,7 @@ log_remove(Pid, ForcedLogs) -> -spec maybe_time(monitor()) -> erlang:timestamp()|no_timing. maybe_time({_Pid, TimingProbability}) -> - case leveled_rand:uniform(100) of + case rand:uniform(100) of N when N =< TimingProbability -> os:timestamp(); _ -> @@ -230,16 +232,15 @@ get_defaults() -> init([LogOpts, LogFrequency, LogOrder]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), RandomLogOrder = lists:map( fun({_R, SL}) -> SL end, lists:keysort( 1, lists:map( - fun(L) -> {leveled_rand:uniform(), L} end, + fun(L) -> {rand:uniform(), L} end, LogOrder))), - InitialJitter = leveled_rand:uniform(2 * 1000 * LogFrequency), + InitialJitter = rand:uniform(2 * 1000 * LogFrequency), erlang:send_after(InitialJitter, self(), report_next_stats), {ok, #state{log_frequency = LogFrequency, log_order = RandomLogOrder}}. diff --git a/src/leveled_pclerk.erl b/src/leveled_pclerk.erl index dcc38124..107bcf15 100644 --- a/src/leveled_pclerk.erl +++ b/src/leveled_pclerk.erl @@ -48,8 +48,8 @@ -define(MIN_TIMEOUT, 200). -define(GROOMING_PERC, 50). --record(state, {owner :: pid() | undefined, - root_path :: string() | undefined, +-record(state, {owner :: pid()|undefined, + root_path :: string()|undefined, pending_deletions = dict:new() :: dict:dict(), sst_options :: sst_options() }). @@ -123,18 +123,20 @@ handle_call(close, _From, State) -> handle_cast(prompt, State) -> handle_info(timeout, State); -handle_cast({push_work, Work}, State) -> +handle_cast( + {push_work, Work}, State = #state{root_path = RP, owner = PCL}) + when ?IS_DEF(RP), is_pid(PCL) -> {ManifestSQN, Deletions} = - handle_work( - Work, - State#state.root_path, State#state.sst_options, State#state.owner), + handle_work(Work, RP, State#state.sst_options, PCL), PDs = dict:store(ManifestSQN, Deletions, State#state.pending_deletions), leveled_log:log(pc022, [ManifestSQN]), {noreply, State#state{pending_deletions = PDs}, ?MIN_TIMEOUT}; -handle_cast({prompt_deletions, ManifestSQN}, State) -> - {Deletions, UpdD} = return_deletions(ManifestSQN, - State#state.pending_deletions), - ok = notify_deletions(Deletions, State#state.owner), +handle_cast( + {prompt_deletions, ManifestSQN}, State = #state{owner = PCL}) + when is_pid(PCL) -> + {Deletions, UpdD} = + return_deletions(ManifestSQN, State#state.pending_deletions), + ok = notify_deletions(Deletions, PCL), {noreply, State#state{pending_deletions = UpdD}, ?MIN_TIMEOUT}; handle_cast({log_level, LogLevel}, State) -> ok = leveled_log:set_loglevel(LogLevel), @@ -152,8 +154,8 @@ handle_cast({remove_logs, ForcedLogs}, State) -> SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, {noreply, State#state{sst_options = SSTopts0}}. -handle_info(timeout, State) -> - ok = leveled_penciller:pcl_workforclerk(State#state.owner), +handle_info(timeout, State = #state{owner = PCL}) when is_pid(PCL) -> + ok = leveled_penciller:pcl_workforclerk(PCL), % When handling work, the clerk can collect a large number of binary % references, so proactively GC this process before receiving any future % work. In under pressure clusters, clerks with large binary memory @@ -207,7 +209,7 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> [SrcLevel + 1, FCnt, MnHBS, MnHS, MnLHS, MnBVHS]) end, SelectMethod = - case leveled_rand:uniform(100) of + case rand:uniform(100) of R when R =< ?GROOMING_PERC -> {grooming, fun grooming_scorer/1}; _ -> @@ -220,16 +222,22 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> leveled_pmanifest:merge_lookup( Manifest, SrcLevel + 1, - Src#manifest_entry.start_key, - Src#manifest_entry.end_key + leveled_pmanifest:entry_startkey(Src), + leveled_pmanifest:entry_endkey(Src) ), Candidates = length(SinkList), leveled_log:log(pc008, [SrcLevel, Candidates]), case Candidates of 0 -> NewLevel = SrcLevel + 1, - leveled_log:log(pc009, [Src#manifest_entry.filename, NewLevel]), - leveled_sst:sst_switchlevels(Src#manifest_entry.owner, NewLevel), + leveled_log:log( + pc009, + [leveled_pmanifest:entry_filename(Src), NewLevel] + ), + leveled_sst:sst_switchlevels( + leveled_pmanifest:entry_owner(Src), + NewLevel + ), Man0 = leveled_pmanifest:switch_manifest_entry( Manifest, @@ -249,7 +257,11 @@ merge(SrcLevel, Manifest, RootPath, OptsSST) -> notify_deletions([], _Penciller) -> ok; notify_deletions([Head|Tail], Penciller) -> - ok = leveled_sst:sst_setfordelete(Head#manifest_entry.owner, Penciller), + ok = + leveled_sst:sst_setfordelete( + leveled_pmanifest:entry_owner(Head), + Penciller + ), notify_deletions(Tail, Penciller). @@ -259,9 +271,12 @@ notify_deletions([Head|Tail], Penciller) -> %% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1 perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN, OptsSST) -> - leveled_log:log(pc010, [Src#manifest_entry.filename, NewSQN]), + leveled_log:log(pc010, [leveled_pmanifest:entry_filename(Src), NewSQN]), SrcList = [{next, Src, all}], - MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), + MaxSQN = + leveled_sst:sst_getmaxsequencenumber( + leveled_pmanifest:entry_owner(Src) + ), SinkLevel = SrcLevel + 1, SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel), Additions = @@ -319,13 +334,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> {ok, Pid, Reply, Bloom} -> {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, Entry = - #manifest_entry{ - start_key=SmallestKey, - end_key=HighestKey, - owner=Pid, - filename=FileName, - bloom=Bloom - }, + leveled_pmanifest:new_entry( + SmallestKey, HighestKey, Pid, FileName, Bloom), leveled_log:log_timer(pc015, [], TS1), do_merge( KL1Rem, KL2Rem, @@ -340,7 +350,8 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, OptsSST, Additions) -> list(leveled_pmanifest:manifest_entry())) -> leveled_pmanifest:manifest_entry(). grooming_scorer([ME | MEs]) -> - InitTombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + InitTombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), {HighestTC, BestME} = grooming_scorer(InitTombCount, ME, MEs), leveled_log:log(pc024, [HighestTC]), BestME. @@ -348,7 +359,8 @@ grooming_scorer([ME | MEs]) -> grooming_scorer(HighestTC, BestME, []) -> {HighestTC, BestME}; grooming_scorer(HighestTC, BestME, [ME | MEs]) -> - TombCount = leveled_sst:sst_gettombcount(ME#manifest_entry.owner), + TombCount = + leveled_sst:sst_gettombcount(leveled_pmanifest:entry_owner(ME)), case TombCount > HighestTC of true -> grooming_scorer(TombCount, ME, MEs); @@ -385,11 +397,17 @@ generate_randomkeys(Count, Acc, BucketLow, BRange) -> BNumber = lists:flatten( io_lib:format("~4..0B", - [BucketLow + leveled_rand:uniform(BRange)])), + [BucketLow + rand:uniform(BRange)])), KNumber = lists:flatten( - io_lib:format("~4..0B", [leveled_rand:uniform(1000)])), - K = {o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, + io_lib:format("~4..0B", [rand:uniform(1000)])), + K = + { + o, + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + null + }, RandKey = {K, {Count + 1, {active, infinity}, leveled_codec:segment_hash(K), @@ -415,7 +433,6 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), {ok, PidL3_1B, _, _} = leveled_sst:sst_newmerge("test/test_area/ledger_files/", @@ -427,7 +444,6 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), {ok, PidL3_2, _, _} = @@ -439,102 +455,116 @@ grooming_score_test() -> 3, 999999, #sst_options{}, - true, true), - {ok, PidL3_2NC, _, _} = - leveled_sst:sst_newmerge("test/test_area/ledger_files/", - "2NC_L3.sst", - KL3_L3, - KL4_L3, - false, - 3, - 999999, - #sst_options{}, - true, - false), - - ME1 = #manifest_entry{owner=PidL3_1}, - ME1B = #manifest_entry{owner=PidL3_1B}, - ME2 = #manifest_entry{owner=PidL3_2}, - ME2NC = #manifest_entry{owner=PidL3_2NC}, + DSK = {o, <<"B">>, <<"SK">>, null}, + DEK = {o, <<"E">>, <<"EK">>, null}, + ME1 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1, "dummyL3_1", none), + ME1B = leveled_pmanifest:new_entry(DSK, DEK, PidL3_1B, "dummyL3_1B", none), + ME2 = leveled_pmanifest:new_entry(DSK, DEK, PidL3_2, "dummyL3_2", none), ?assertMatch(ME1, grooming_scorer([ME1, ME2])), ?assertMatch(ME1, grooming_scorer([ME2, ME1])), % prefer the file with the tombstone - ?assertMatch(ME2NC, grooming_scorer([ME1, ME2NC])), - ?assertMatch(ME2NC, grooming_scorer([ME2NC, ME1])), - % not_counted > 1 - we will merge files in unexpected (i.e. legacy) - % format first ?assertMatch(ME1B, grooming_scorer([ME1B, ME2])), ?assertMatch(ME2, grooming_scorer([ME2, ME1B])), % If the file with the tombstone is in the basement, it will have % no tombstone so the first file will be chosen lists:foreach(fun(P) -> leveled_sst:sst_clear(P) end, - [PidL3_1, PidL3_1B, PidL3_2, PidL3_2NC]). + [PidL3_1, PidL3_1B, PidL3_2]). merge_file_test() -> ok = filelib:ensure_dir("test/test_area/ledger_files/"), KL1_L1 = lists:sort(generate_randomkeys(8000, 0, 1000)), {ok, PidL1_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L1.sst", - 1, - KL1_L1, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L1.sst", + 1, + KL1_L1, + 999999, + #sst_options{} + ), KL1_L2 = lists:sort(generate_randomkeys(8000, 0, 250)), {ok, PidL2_1, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL1_L2.sst", - 2, - KL1_L2, - 999999, - #sst_options{}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL1_L2.sst", + 2, + KL1_L2, + 999999, + #sst_options{} + ), KL2_L2 = lists:sort(generate_randomkeys(8000, 250, 250)), {ok, PidL2_2, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL2_L2.sst", - 2, - KL2_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL2_L2.sst", + 2, + KL2_L2, + 999999, + #sst_options{press_method = lz4} + ), KL3_L2 = lists:sort(generate_randomkeys(8000, 500, 250)), {ok, PidL2_3, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL3_L2.sst", - 2, - KL3_L2, - 999999, - #sst_options{press_method = lz4}), + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL3_L2.sst", + 2, + KL3_L2, + 999999, + #sst_options{press_method = lz4} + ), KL4_L2 = lists:sort(generate_randomkeys(8000, 750, 250)), {ok, PidL2_4, _, _} = - leveled_sst:sst_new("test/test_area/ledger_files/", - "KL4_L2.sst", - 2, - KL4_L2, - 999999, - #sst_options{press_method = lz4}), - E1 = #manifest_entry{owner = PidL1_1, - filename = "./KL1_L1.sst", - end_key = lists:last(KL1_L1), - start_key = lists:nth(1, KL1_L1)}, - E2 = #manifest_entry{owner = PidL2_1, - filename = "./KL1_L2.sst", - end_key = lists:last(KL1_L2), - start_key = lists:nth(1, KL1_L2)}, - E3 = #manifest_entry{owner = PidL2_2, - filename = "./KL2_L2.sst", - end_key = lists:last(KL2_L2), - start_key = lists:nth(1, KL2_L2)}, - E4 = #manifest_entry{owner = PidL2_3, - filename = "./KL3_L2.sst", - end_key = lists:last(KL3_L2), - start_key = lists:nth(1, KL3_L2)}, - E5 = #manifest_entry{owner = PidL2_4, - filename = "./KL4_L2.sst", - end_key = lists:last(KL4_L2), - start_key = lists:nth(1, KL4_L2)}, + leveled_sst:sst_new( + "test/test_area/ledger_files/", + "KL4_L2.sst", + 2, + KL4_L2, + 999999, + #sst_options{press_method = lz4} + ), + E1 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L1), + lists:last(KL1_L1), + PidL1_1, + "./KL1_L1.sst", + none + ), + E2 = + leveled_pmanifest:new_entry( + lists:nth(1, KL1_L2), + lists:last(KL1_L2), + PidL2_1, + "./KL1_L2.sst", + none + ), + E3 = + leveled_pmanifest:new_entry( + lists:nth(1, KL2_L2), + lists:last(KL2_L2), + PidL2_2, + "./KL2_L2.sst", + none + ), + E4 = + leveled_pmanifest:new_entry( + lists:nth(1, KL3_L2), + lists:last(KL3_L2), + PidL2_3, + "./KL3_L2.sst", + none + ), + E5 = + leveled_pmanifest:new_entry( + lists:nth(1, KL4_L2), + lists:last(KL4_L2), + PidL2_4, + "./KL4_L2.sst", + none + ), Man0 = leveled_pmanifest:new_manifest(), Man1 = leveled_pmanifest:insert_manifest_entry(Man0, 1, 2, E2), diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index 64c325a2..0a432ae3 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -202,6 +202,10 @@ -export([pcl_getsstpids/1, pcl_getclerkpid/1]). +% Test functions to ignore for equalizer +-eqwalizer({nowarn_function, fetch_status_test/0}). +-eqwalizer({nowarn_function, maybe_pause_push/2}). + -ifdef(TEST). -export([clean_testdir/1]). -endif. @@ -226,6 +230,8 @@ -record(state, {manifest :: leveled_pmanifest:manifest() | undefined | redacted, + % Can be undefined in some snapshots, but must always be + % defined when using a primary penciller query_manifest :: {list(), leveled_codec:ledger_key(), @@ -233,6 +239,7 @@ % Slimmed down version of the manifest containing part % related to specific query, and the StartKey/EndKey % used to extract this part + % Only found in snapshots persisted_sqn = 0 :: integer(), % The highest SQN persisted ledger_sqn = 0 :: integer(), % The highest SQN added to L0 @@ -241,7 +248,8 @@ levelzero_constructor :: pid() | undefined, levelzero_cache = [] :: levelzero_cache() | redacted, levelzero_size = 0 :: integer(), - levelzero_maxcachesize :: integer() | undefined, + levelzero_maxcachesize = 0 :: non_neg_integer(), + % Will default to 0 in snapshots (when not required) levelzero_cointoss = false :: boolean(), levelzero_index :: leveled_pmem:index_array() | undefined | redacted, @@ -249,6 +257,7 @@ root_path = "test" :: string(), clerk :: pid() | undefined, + % Can only be undefined in a snapshot is_snapshot = false :: boolean(), snapshot_fully_loaded = false :: boolean(), @@ -274,14 +283,16 @@ -type penciller_options() :: #penciller_options{}. --type bookies_memory() :: {tuple()|empty_cache, - array:array()|empty_array, - integer()|infinity, - integer()}. +-type bookies_memory() :: + { + ets:table()|tuple()|empty_cache, + array:array()|empty_array, + integer()|infinity, + integer() + }. -type pcl_state() :: #state{}. -type levelzero_cacheentry() :: {pos_integer(), leveled_tree:leveled_tree()}. -type levelzero_cache() :: list(levelzero_cacheentry()). --type bad_ledgerkey() :: list(). -type sqn_check() :: current|replaced|missing. -type sst_fetchfun() :: fun((pid(), @@ -291,9 +302,9 @@ leveled_codec:ledger_kv()|not_present). -type levelzero_returnfun() :: fun((levelzero_cacheentry()) -> ok). -type pclacc_fun() :: - fun((leveled_codec:ledger_key(), + fun((leveled_codec:object_key(), leveled_codec:ledger_value(), - term()) -> term()). + dynamic()) -> dynamic()). -type sst_options() :: #sst_options{}. -export_type( @@ -318,13 +329,17 @@ %% query is run against the level zero space and just the query results are %% copied into the clone. pcl_start(PCLopts) -> - gen_server:start_link(?MODULE, [leveled_log:get_opts(), PCLopts], []). + {ok, Pcl} = + gen_server:start_link(?MODULE, [leveled_log:get_opts(), PCLopts], []), + {ok, Pcl}. -spec pcl_snapstart(penciller_options()) -> {ok, pid()}. %% @doc %% Don't link to the bookie - this is a snpashot pcl_snapstart(PCLopts) -> - gen_server:start(?MODULE, [leveled_log:get_opts(), PCLopts], []). + {ok, PclSnap} = + gen_server:start(?MODULE, [leveled_log:get_opts(), PCLopts], []), + {ok, PclSnap}. -spec pcl_pushmem(pid(), bookies_memory()) -> ok|returned. %% @doc @@ -371,16 +386,20 @@ pcl_fetchlevelzero(Pid, Slot, ReturnFun) -> boolean()) -> leveled_codec:ledger_kv()|not_present. %% @doc %% Fetch a key, return the first (highest SQN) occurrence of that Key along -%% with the value. +%% with the value. %% %% Hash should be result of leveled_codec:segment_hash(Key) +%% The L0Index cannot be used when in head_only mode - as although such keys +%% are fetchable no index entries are created whne added to the ledger cache pcl_fetch(Pid, Key, Hash, UseL0Index) -> gen_server:call(Pid, {fetch, Key, Hash, UseL0Index}, infinity). --spec pcl_fetchkeys(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any(), as_pcl|by_runner) -> any(). +-spec pcl_fetchkeys( + pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic()) -> dynamic(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This will cover %% all keys in the range - so must only be run against snapshots of the @@ -393,6 +412,19 @@ pcl_fetch(Pid, Key, Hash, UseL0Index) -> pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc) -> pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, as_pcl). +-spec pcl_fetchkeys + (pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic(), + as_pcl) -> dynamic(); + (pid(), + leveled_codec:query_key(), + leveled_codec:query_key(), + pclacc_fun(), + dynamic(), + by_runner) -> fun(() -> dynamic()). pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, By) -> gen_server:call(Pid, {fetch_keys, @@ -403,13 +435,14 @@ pcl_fetchkeys(Pid, StartKey, EndKey, AccFun, InitAcc, By) -> infinity). --spec pcl_fetchkeysbysegment(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any(), - leveled_codec:segment_list(), - false | leveled_codec:lastmod_range(), - boolean()) -> any(). +-spec pcl_fetchkeysbysegment( + pid(), + leveled_codec:ledger_key(), + leveled_codec:ledger_key(), + pclacc_fun(), any(), + leveled_codec:segment_list(), + false | leveled_codec:lastmod_range(), + boolean()) -> any(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This will cover %% all keys in the range - so must only be run against snapshots of the @@ -440,26 +473,26 @@ pcl_fetchkeysbysegment(Pid, StartKey, EndKey, AccFun, InitAcc, by_runner}, infinity). --spec pcl_fetchnextkey(pid(), - leveled_codec:ledger_key(), - leveled_codec:ledger_key(), - pclacc_fun(), any()) -> any(). +-spec pcl_fetchnextkey( + pid(), + leveled_codec:ledger_key(), + leveled_codec:ledger_key(), + pclacc_fun(), any()) -> any(). %% @doc %% Run a range query between StartKey and EndKey (inclusive). This has the %% same constraints as pcl_fetchkeys/5, but will only return the first key %% found in erlang term order. pcl_fetchnextkey(Pid, StartKey, EndKey, AccFun, InitAcc) -> - gen_server:call(Pid, - {fetch_keys, - StartKey, EndKey, - AccFun, InitAcc, - false, false, 1, - as_pcl}, - infinity). - --spec pcl_checksequencenumber(pid(), - leveled_codec:ledger_key()|bad_ledgerkey(), - integer()) -> sqn_check(). + gen_server:call( + Pid, + {fetch_keys, + StartKey, EndKey, AccFun, InitAcc, false, false, 1, as_pcl + }, + infinity + ). + +-spec pcl_checksequencenumber( + pid(), leveled_codec:ledger_key(), integer()) -> sqn_check(). %% @doc %% Check if the sequence number of the passed key is not replaced by a change %% after the passed sequence number. Will return: @@ -598,13 +631,13 @@ pcl_checkforwork(Pid) -> pcl_loglevel(Pid, LogLevel) -> gen_server:cast(Pid, {log_level, LogLevel}). --spec pcl_addlogs(pid(), list(string())) -> ok. +-spec pcl_addlogs(pid(), list(atom())) -> ok. %% @doc %% Add to the list of forced logs, a list of more forced logs pcl_addlogs(Pid, ForcedLogs) -> gen_server:cast(Pid, {add_logs, ForcedLogs}). --spec pcl_removelogs(pid(), list(string())) -> ok. +-spec pcl_removelogs(pid(), list(atom())) -> ok. %% @doc %% Remove from the list of forced logs, a list of forced logs pcl_removelogs(Pid, ForcedLogs) -> @@ -628,13 +661,14 @@ pcl_getclerkpid(Pid) -> init([LogOpts, PCLopts]) -> leveled_log:save(LogOpts), - leveled_rand:seed(), case {PCLopts#penciller_options.root_path, PCLopts#penciller_options.start_snapshot, PCLopts#penciller_options.snapshot_query, - PCLopts#penciller_options.bookies_mem} of - {undefined, _Snapshot=true, Query, BookiesMem} -> - SrcPenciller = PCLopts#penciller_options.source_penciller, + PCLopts#penciller_options.bookies_mem, + PCLopts#penciller_options.source_penciller + } of + {undefined, _Snapshot=true, Query, BookiesMem, SrcPenciller} + when ?IS_DEF(BookiesMem), ?IS_DEF(SrcPenciller) -> LongRunning = PCLopts#penciller_options.snapshot_longrunning, %% monitor the bookie, and close the snapshot when bookie %% exits @@ -650,7 +684,7 @@ init([LogOpts, PCLopts]) -> clerk = undefined, bookie_monref = BookieMonitor, source_penciller = SrcPenciller}}; - {_RootPath, _Snapshot=false, _Q, _BM} -> + {_RootPath, _Snapshot=false, _Q, _BM, _SP} -> start_from_file(PCLopts) end. @@ -722,7 +756,9 @@ handle_call({push_mem, {LedgerTable, PushedIdx, MinSQN, MaxSQN}}, ledger_sqn = UpdMaxSQN}} end end; -handle_call({fetch, Key, Hash, UseL0Index}, _From, State) -> +handle_call( + {fetch, Key, Hash, UseL0Index}, _From, State = #state{manifest = M}) + when ?IS_DEF(M) -> L0Idx = case UseL0Index of true -> @@ -732,20 +768,25 @@ handle_call({fetch, Key, Hash, UseL0Index}, _From, State) -> end, R = timed_fetch_mem( - Key, Hash, State#state.manifest, - State#state.levelzero_cache, L0Idx, - State#state.monitor), + Key, + Hash, + M, + State#state.levelzero_cache, + L0Idx, + State#state.monitor + ), {reply, R, State}; -handle_call({check_sqn, Key, Hash, SQN}, _From, State) -> +handle_call( + {check_sqn, Key, Hash, SQN}, + _From, + State = #state{manifest = M, levelzero_cache = L0C, levelzero_index = L0I}) + % This is either a primary penciller, or snapshot taken without a query + % so that it contains the full level zero. + % Not to be used in head_only mode (where levelzero_index may not be + % complete) + when ?IS_DEF(M), ?IS_DEF(L0C), ?IS_DEF(L0I) -> {reply, - compare_to_sqn( - fetch_sqn( - Key, - Hash, - State#state.manifest, - State#state.levelzero_cache, - State#state.levelzero_index), - SQN), + compare_to_sqn(fetch_sqn(Key, Hash, M, L0C, L0I), SQN), State}; handle_call({fetch_keys, StartKey, EndKey, @@ -786,7 +827,10 @@ handle_call({fetch_keys, leveled_sst:extract_hash( leveled_codec:strip_to_segmentonly(LKV)), case CheckSeg of - CheckSeg when CheckSeg >= Min, CheckSeg =< Max -> + CheckSeg + when is_integer(CheckSeg), + CheckSeg >= Min, + CheckSeg =< Max -> CheckFun(CheckSeg); _ -> false @@ -827,8 +871,11 @@ handle_call({fetch_keys, end; handle_call(get_startup_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; -handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, - _From, State) -> +handle_call( + {register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, + _From, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> % Register and load a snapshot % % For setup of the snapshot to be efficient should pass a query @@ -843,8 +890,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, false -> State#state.snaptimeout_short end, - Manifest0 = - leveled_pmanifest:add_snapshot(State#state.manifest, Snapshot, TimeO), + Manifest0 = leveled_pmanifest:add_snapshot(Manifest, Snapshot, TimeO), {BookieIncrTree, BookieIdx, MinSQN, MaxSQN} = BookiesMem, LM1Cache = @@ -869,7 +915,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, ledger_sqn = UpdMaxSQN, levelzero_size = UpdSize, persisted_sqn = State#state.persisted_sqn}, - leveled_pmanifest:copy_manifest(State#state.manifest), + leveled_pmanifest:copy_manifest(Manifest), undefined}; {StartKey, EndKey} -> SW = os:timestamp(), @@ -885,7 +931,7 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, persisted_sqn = State#state.persisted_sqn}, undefined, {leveled_pmanifest:query_manifest( - State#state.manifest, StartKey, EndKey), + Manifest, StartKey, EndKey), StartKey, EndKey}}; undefined -> @@ -911,20 +957,29 @@ handle_call({register_snapshot, Snapshot, Query, BookiesMem, LongRunning}, levelzero_size = UpdSize, ledger_sqn = UpdMaxSQN, persisted_sqn = State#state.persisted_sqn}, - leveled_pmanifest:copy_manifest(State#state.manifest), + leveled_pmanifest:copy_manifest(Manifest), undefined} end, {reply, {ok, - CloneState#state{snapshot_fully_loaded = true, - snapshot_time = leveled_util:integer_now(), - manifest = ManifestClone, - query_manifest = QueryManifest}}, + CloneState#state{ + snapshot_fully_loaded = true, + snapshot_time = leveled_util:integer_now(), + manifest = ManifestClone, + query_manifest = QueryManifest + } + }, State#state{manifest = Manifest0}}; handle_call(close, _From, State=#state{is_snapshot=Snap}) when Snap == true -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), {stop, normal, ok, State}; -handle_call(close, From, State) -> +handle_call( + close, + From, + State = #state{manifest = Manifest, clerk = Clerk, levelzero_cache = L0C}) + % By definition not a snapshot (as snapshot covered by clause above), + % so manifest, clerk and cache must all be present + when ?IS_DEF(Manifest), ?IS_DEF(Clerk), ?IS_DEF(L0C) -> % Level 0 files lie outside of the manifest, and so if there is no L0 % file present it is safe to write the current contents of memory. If % there is a L0 file present - then the memory can be dropped (it is @@ -934,19 +989,18 @@ handle_call(close, From, State) -> % % The penciller should close each file in the manifest, and call a close % on the clerk. - ok = leveled_pclerk:clerk_close(State#state.clerk), + ok = leveled_pclerk:clerk_close(Clerk), leveled_log:log(p0008, [close]), L0Left = State#state.levelzero_size > 0, case (not State#state.levelzero_pending and L0Left) of true -> - Man0 = State#state.manifest, {Constructor, _} = roll_memory( - leveled_pmanifest:get_manifest_sqn(Man0) + 1, + leveled_pmanifest:get_manifest_sqn(Manifest) + 1, State#state.ledger_sqn, State#state.root_path, - State#state.levelzero_cache, - length(State#state.levelzero_cache), + L0C, + length(L0C), State#state.sst_options, true), ok = leveled_sst:sst_close(Constructor); @@ -955,49 +1009,59 @@ handle_call(close, From, State) -> end, gen_server:cast(self(), {maybe_defer_shutdown, close, From}), {noreply, State}; -handle_call(doom, From, State) -> +handle_call( + doom, From, State = #state{clerk = Clerk}) + when ?IS_DEF(Clerk) -> leveled_log:log(p0030, []), - ok = leveled_pclerk:clerk_close(State#state.clerk), + ok = leveled_pclerk:clerk_close(Clerk), gen_server:cast(self(), {maybe_defer_shutdown, doom, From}), {noreply, State}; -handle_call({checkbloom_fortest, Key, Hash}, _From, State) -> - Manifest = State#state.manifest, +handle_call( + {checkbloom_fortest, Key, Hash}, _From, State = #state{manifest = Man}) + when ?IS_DEF(Man) -> FoldFun = fun(Level, Acc) -> case Acc of true -> true; false -> - case leveled_pmanifest:key_lookup(Manifest, Level, Key) of + case leveled_pmanifest:key_lookup(Man, Level, Key) of false -> false; FP -> - leveled_pmanifest:check_bloom(Manifest, FP, Hash) + leveled_pmanifest:check_bloom(Man, FP, Hash) end end end, {reply, lists:foldl(FoldFun, false, lists:seq(0, ?MAX_LEVELS)), State}; -handle_call(check_for_work, _From, State) -> - {_WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest), +handle_call( + check_for_work, _From, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + {_WL, WC} = leveled_pmanifest:check_for_work(Manifest), {reply, WC > 0, State}; handle_call(persisted_sqn, _From, State) -> {reply, State#state.persisted_sqn, State}; -handle_call(get_sstpids, _From, State) -> - {reply, leveled_pmanifest:get_sstpids(State#state.manifest), State}; +handle_call( + get_sstpids, _From, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + {reply, leveled_pmanifest:get_sstpids(Manifest), State}; handle_call(get_clerkpid, _From, State) -> {reply, State#state.clerk, State}. -handle_cast({manifest_change, Manifest}, State) -> +handle_cast( + {manifest_change, Manifest}, + State = #state{manifest = OldManifest, clerk = Clerk}) + when ?IS_DEF(OldManifest), ?IS_DEF(Clerk) -> NewManSQN = leveled_pmanifest:get_manifest_sqn(Manifest), - OldManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest), + OldManSQN = leveled_pmanifest:get_manifest_sqn(OldManifest), leveled_log:log(p0041, [OldManSQN, NewManSQN]), % Only safe to update the manifest if the SQN increments if NewManSQN > OldManSQN -> ok = - leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN), + leveled_pclerk:clerk_promptdeletions(Clerk, NewManSQN), % This is accepted as the new manifest, files may be deleted UpdManifest0 = - leveled_pmanifest:merge_snapshot(State#state.manifest, Manifest), + leveled_pmanifest:merge_snapshot(OldManifest, Manifest), % Need to preserve the penciller's view of snapshots stored in % the manifest UpdManifest1 = @@ -1012,9 +1076,11 @@ handle_cast({manifest_change, Manifest}, State) -> maybe_release = false, work_ongoing=false}} end; -handle_cast({release_snapshot, Snapshot}, State) -> +handle_cast( + {release_snapshot, Snapshot}, State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> Manifest0 = - leveled_pmanifest:release_snapshot(State#state.manifest, Snapshot), + leveled_pmanifest:release_snapshot(Manifest, Snapshot), leveled_log:log(p0003, [Snapshot]), {noreply, State#state{manifest=Manifest0}}; handle_cast({confirm_delete, PDFN, FilePid}, State=#state{is_snapshot=Snap}) @@ -1066,19 +1132,17 @@ handle_cast({confirm_delete, PDFN, FilePid}, State=#state{is_snapshot=Snap}) State#state{manifest = UpdManifest}} end end; -handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> +handle_cast( + {levelzero_complete, FN, StartKey, EndKey, Bloom}, + State = #state{manifest = Man, levelzero_constructor = L0C, clerk = Clerk}) + when ?IS_DEF(Man), ?IS_DEF(L0C), ?IS_DEF(Clerk) -> leveled_log:log(p0029, []), - ManEntry = #manifest_entry{start_key=StartKey, - end_key=EndKey, - owner=State#state.levelzero_constructor, - filename=FN, - bloom=Bloom}, - ManifestSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1, + ManEntry = leveled_pmanifest:new_entry(StartKey, EndKey, L0C, FN, Bloom), + ManifestSQN = leveled_pmanifest:get_manifest_sqn(Man) + 1, UpdMan = - leveled_pmanifest:insert_manifest_entry( - State#state.manifest, ManifestSQN, 0, ManEntry), + leveled_pmanifest:insert_manifest_entry(Man, ManifestSQN, 0, ManEntry), % Prompt clerk to ask about work - do this for every L0 roll - ok = leveled_pclerk:clerk_prompt(State#state.clerk), + ok = leveled_pclerk:clerk_prompt(Clerk), {noreply, State#state{levelzero_cache=[], levelzero_index=[], levelzero_pending=false, @@ -1086,17 +1150,19 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey, Bloom}, State) -> levelzero_size=0, manifest=UpdMan, persisted_sqn=State#state.ledger_sqn}}; -handle_cast(work_for_clerk, State) -> +handle_cast( + work_for_clerk, + State = #state{manifest = Man, levelzero_cache = L0Cache, clerk = Clerk}) + when ?IS_DEF(Man), ?IS_DEF(L0Cache), ?IS_DEF(Clerk) -> case {(State#state.levelzero_pending or State#state.work_ongoing), - leveled_pmanifest:levelzero_present(State#state.manifest)} of + leveled_pmanifest:levelzero_present(Man)} of {true, _L0Present} -> % Work is blocked by ongoing activity {noreply, State}; {false, true} -> % If L0 present, and no work ongoing - dropping L0 to L1 is the % priority - ok = leveled_pclerk:clerk_push( - State#state.clerk, {0, State#state.manifest}), + ok = leveled_pclerk:clerk_push(Clerk, {0, Man}), {noreply, State#state{work_ongoing=true}}; {false, false} -> % No impediment to work - see what other work may be required @@ -1106,10 +1172,9 @@ handle_cast(work_for_clerk, State) -> State#state.levelzero_size, State#state.levelzero_maxcachesize, State#state.levelzero_cointoss), - CacheAlreadyFull = - leveled_pmem:cache_full(State#state.levelzero_cache), + CacheAlreadyFull = leveled_pmem:cache_full(L0Cache), % Check for a backlog of work - {WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest), + {WL, WC} = leveled_pmanifest:check_for_work(Man), case {WC, (CacheAlreadyFull or CacheOverSize)} of {0, false} -> % No work required @@ -1119,15 +1184,14 @@ handle_cast(work_for_clerk, State) -> % Must not do this if there is a work backlog beyond the % tolerance, as then the backlog may never be addressed. NextSQN = - leveled_pmanifest:get_manifest_sqn( - State#state.manifest) + 1, + leveled_pmanifest:get_manifest_sqn(Man) + 1, {Constructor, none} = roll_memory( NextSQN, State#state.ledger_sqn, State#state.root_path, none, - length(State#state.levelzero_cache), + length(L0Cache), State#state.sst_options, false), {noreply, @@ -1142,15 +1206,16 @@ handle_cast(work_for_clerk, State) -> Backlog = WC >= ?WORKQUEUE_BACKLOG_TOLERANCE, leveled_log:log(p0024, [WC, Backlog, L0Full]), [TL|_Tail] = WL, - ok = - leveled_pclerk:clerk_push( - State#state.clerk, {TL, State#state.manifest}), + ok = leveled_pclerk:clerk_push(Clerk, {TL, Man}), {noreply, State#state{ work_backlog = Backlog, work_ongoing = true}} end end; -handle_cast({fetch_levelzero, Slot, ReturnFun}, State) -> +handle_cast( + {fetch_levelzero, Slot, ReturnFun}, + State = #state{levelzero_cache = L0Cache}) + when ?IS_DEF(L0Cache) -> ReturnFun(lists:nth(Slot, State#state.levelzero_cache)), {noreply, State}; handle_cast({log_level, LogLevel}, State) -> @@ -1173,8 +1238,11 @@ handle_cast({remove_logs, ForcedLogs}, State) -> SSTopts = State#state.sst_options, SSTopts0 = SSTopts#sst_options{log_options = leveled_log:get_opts()}, {noreply, State#state{sst_options = SSTopts0}}; -handle_cast({maybe_defer_shutdown, ShutdownType, From}, State) -> - case length(leveled_pmanifest:snapshot_pids(State#state.manifest)) of +handle_cast( + {maybe_defer_shutdown, ShutdownType, From}, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> + case length(leveled_pmanifest:snapshot_pids(Manifest)) of 0 -> gen_server:cast(self(), {complete_shutdown, ShutdownType, From}), {noreply, State}; @@ -1195,11 +1263,14 @@ handle_cast({maybe_defer_shutdown, ShutdownType, From}, State) -> {noreply, State} end end; -handle_cast({complete_shutdown, ShutdownType, From}, State) -> +handle_cast( + {complete_shutdown, ShutdownType, From}, + State = #state{manifest = Manifest}) + when ?IS_DEF(Manifest) -> lists:foreach( fun(Snap) -> ok = pcl_snapclose(Snap) end, - leveled_pmanifest:snapshot_pids(State#state.manifest)), - shutdown_manifest(State#state.manifest, State#state.levelzero_constructor), + leveled_pmanifest:snapshot_pids(Manifest)), + shutdown_manifest(Manifest, State#state.levelzero_constructor), case ShutdownType of doom -> ManifestFP = State#state.root_path ++ "/" ++ ?MANIFEST_FP ++ "/", @@ -1211,8 +1282,10 @@ handle_cast({complete_shutdown, ShutdownType, From}, State) -> {stop, normal, State}. %% handle the bookie stopping and stop this snapshot -handle_info({'DOWN', BookieMonRef, process, _BookiePid, _Info}, - State=#state{bookie_monref = BookieMonRef}) -> +handle_info( + {'DOWN', BookieMonRef, process, _BookiePid, _Info}, + State=#state{bookie_monref = BookieMonRef, source_penciller = SrcPCL}) + when ?IS_DEF(SrcPCL) -> ok = pcl_releasesnapshot(State#state.source_penciller, self()), {stop, normal, State}; handle_info(_Info, State) -> @@ -1262,7 +1335,12 @@ sst_filename(ManSQN, Level, Count) -> %%% Internal functions %%%============================================================================ --spec update_clerk(pid()|undefined, fun((pid(), term()) -> ok), term()) -> ok. +-type update_forcedlogs_fun() :: fun((pid(), list(atom())) -> ok). +-type update_loglevel_fun() :: fun((pid(), atom()) -> ok). + +-spec update_clerk + (pid()|undefined, update_loglevel_fun(), atom()) -> ok; + (pid()|undefined, update_forcedlogs_fun(), list(atom())) -> ok. update_clerk(undefined, _F, _T) -> ok; update_clerk(Clerk, F, T) when is_pid(Clerk) -> @@ -1272,7 +1350,12 @@ update_clerk(Clerk, F, T) when is_pid(Clerk) -> %% @doc %% Normal start of a penciller (i.e. not a snapshot), needs to read the %% filesystem and reconstruct the ledger from the files that it finds -start_from_file(PCLopts) -> +start_from_file( + PCLopts = + #penciller_options{ + root_path = RootPath, max_inmemory_tablesize = MaxTableSize} + ) + when ?IS_DEF(RootPath), ?IS_DEF(MaxTableSize) -> RootPath = PCLopts#penciller_options.root_path, MaxTableSize = PCLopts#penciller_options.max_inmemory_tablesize, OptsSST = PCLopts#penciller_options.sst_options, @@ -1288,25 +1371,13 @@ start_from_file(PCLopts) -> % vnode syncronisation issues (e.g. stop them all by default merging to % level zero concurrently) - InitState = - #state{ - clerk = MergeClerk, - root_path = RootPath, - levelzero_maxcachesize = MaxTableSize, - levelzero_cointoss = CoinToss, - levelzero_index = [], - snaptimeout_short = SnapTimeoutShort, - snaptimeout_long = SnapTimeoutLong, - sst_options = OptsSST, - monitor = Monitor}, - %% Open manifest Manifest0 = leveled_pmanifest:open_manifest(RootPath), OpenFun = fun(FN, Level) -> {ok, Pid, {_FK, _LK}, Bloom} = - leveled_sst:sst_open(sst_rootpath(RootPath), - FN, OptsSST, Level), + leveled_sst:sst_open( + sst_rootpath(RootPath), FN, OptsSST, Level), {Pid, Bloom} end, SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, @@ -1317,7 +1388,7 @@ start_from_file(PCLopts) -> leveled_log:log(p0035, [ManSQN]), %% Find any L0 files L0FN = sst_filename(ManSQN + 1, 0, 0), - {State0, FileList0} = + {{InitManifest, InitLedgerSQN, InitPersistSQN}, FileList0} = case filelib:is_file(filename:join(sst_rootpath(RootPath), L0FN)) of true -> leveled_log:log(p0015, [L0FN]), @@ -1327,32 +1398,39 @@ start_from_file(PCLopts) -> {ok, L0Pid, {L0StartKey, L0EndKey}, Bloom} = L0Open, L0SQN = leveled_sst:sst_getmaxsequencenumber(L0Pid), L0Entry = - #manifest_entry{ - start_key = L0StartKey, - end_key = L0EndKey, - filename = L0FN, - owner = L0Pid, - bloom = Bloom}, + leveled_pmanifest:new_entry( + L0StartKey, L0EndKey, L0Pid, L0FN, Bloom), Manifest2 = leveled_pmanifest:insert_manifest_entry( Manifest1, ManSQN + 1, 0, L0Entry), leveled_log:log(p0016, [L0SQN]), LedgerSQN = max(MaxSQN, L0SQN), - {InitState#state{ - manifest = Manifest2, - ledger_sqn = LedgerSQN, - persisted_sqn = LedgerSQN}, - [L0FN|FileList]}; + { + {Manifest2, LedgerSQN, LedgerSQN}, + [L0FN|FileList] + }; false -> leveled_log:log(p0017, []), - {InitState#state{ - manifest = Manifest1, - ledger_sqn = MaxSQN, - persisted_sqn = MaxSQN}, - FileList} + {{Manifest1, MaxSQN, MaxSQN}, FileList} end, ok = archive_files(RootPath, FileList0), - {ok, State0}. + { + ok, + #state{ + clerk = MergeClerk, + root_path = RootPath, + levelzero_maxcachesize = MaxTableSize, + levelzero_cointoss = CoinToss, + levelzero_index = [], + snaptimeout_short = SnapTimeoutShort, + snaptimeout_long = SnapTimeoutLong, + sst_options = OptsSST, + monitor = Monitor, + manifest = InitManifest, + ledger_sqn = InitLedgerSQN, + persisted_sqn = InitPersistSQN + } + }. -spec shutdown_manifest(leveled_pmanifest:manifest(), pid()|undefined) -> ok. @@ -1362,13 +1440,13 @@ shutdown_manifest(Manifest, L0Constructor) -> EntryCloseFun = fun(ME) -> Owner = - case is_record(ME, manifest_entry) of + case leveled_pmanifest:is_entry(ME) of true -> - ME#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME); false -> case ME of {_SK, ME0} -> - ME0#manifest_entry.owner; + leveled_pmanifest:entry_owner(ME0); ME -> ME end @@ -1442,7 +1520,7 @@ maybe_cache_too_big(NewL0Size, L0MaxSize, CoinToss) -> RandomFactor = case CoinToss of true -> - case leveled_rand:uniform(?COIN_SIDECOUNT) of + case rand:uniform(?COIN_SIDECOUNT) of 1 -> true; _ -> @@ -1482,7 +1560,9 @@ roll_memory(NextManSQN, LedgerSQN, RootPath, none, CL, SSTOpts, false) -> leveled_sst:sst_newlevelzero( L0Path, L0FN, CL, FetchFun, PCL, LedgerSQN, SSTOpts), {Constructor, none}; -roll_memory(NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) -> +roll_memory( + NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) + when is_list(L0Cache) -> L0Path = sst_rootpath(RootPath), L0FN = sst_filename(NextManSQN, 0, 0), FetchFun = fun(Slot) -> lists:nth(Slot, L0Cache) end, @@ -1495,7 +1575,8 @@ roll_memory(NextManSQN, LedgerSQN, RootPath, L0Cache, CL, SSTOpts, true) -> -spec timed_fetch_mem( tuple(), {integer(), integer()}, - leveled_pmanifest:manifest(), list(), + leveled_pmanifest:manifest(), + list(), leveled_pmem:index_array(), leveled_monitor:monitor()) -> leveled_codec:ledger_kv()|not_found. %% @doc @@ -1622,9 +1703,11 @@ compare_to_sqn(Obj, SQN) -> boolean()) -> ok. maybelog_fetch_timing(_Monitor, _Level, no_timing, _NF) -> ok; -maybelog_fetch_timing({Pid, _StatsFreq}, _Level, FetchTime, true) -> +maybelog_fetch_timing( + {Pid, _StatsFreq}, _Level, FetchTime, true) when is_pid(Pid) -> leveled_monitor:add_stat(Pid, {pcl_fetch_update, not_found, FetchTime}); -maybelog_fetch_timing({Pid, _StatsFreq}, Level, FetchTime, _NF) -> +maybelog_fetch_timing( + {Pid, _StatsFreq}, Level, FetchTime, _NF) when is_pid(Pid) -> leveled_monitor:add_stat(Pid, {pcl_fetch_update, Level, FetchTime}). %%%============================================================================ @@ -1758,12 +1841,13 @@ find_nextkeys( {no_more_keys, FoundKVs}; find_nextkeys( Iter, {[], {BKL, BestKV}}, FoundKVs, _Ls, {W, _SW}, _SearchInfo) - when length(FoundKVs) == W - 1 -> + when length(FoundKVs) == W - 1, BestKV =/= null -> % All levels scanned, and there are now W keys (W - 1 previously found plus % the latest best key) {maps:update_with(BKL, fun tl/1, Iter), [BestKV|FoundKVs]}; find_nextkeys( - Iter, {[], {BKL, BestKV}}, FoundKVs, Ls, BatchInfo, SearchInfo) -> + Iter, {[], {BKL, BestKV}}, FoundKVs, Ls, BatchInfo, SearchInfo) + when BestKV =/= null -> % All levels scanned so this is the best key ... now loop to find more find_nextkeys( maps:update_with(BKL, fun tl/1, Iter), @@ -1828,7 +1912,7 @@ find_nextkeys( {OtherLevels, PrevBest}, FoundKVs, Ls, BI, SI); - [{Key, Val}|_RestOfKeys] -> + [{Key, Val}|_RestOfKeys] when BKV =/= null -> case leveled_codec:key_dominates({Key, Val}, BKV) of true -> find_nextkeys( @@ -1897,15 +1981,18 @@ generate_randomkeys({Count, StartSQN}) -> generate_randomkeys(0, _SQN, Acc) -> lists:reverse(Acc); generate_randomkeys(Count, SQN, Acc) -> - K = {o, - lists:concat(["Bucket", leveled_rand:uniform(1024)]), - lists:concat(["Key", leveled_rand:uniform(1024)]), - null}, - RandKey = {K, - {SQN, - {active, infinity}, - leveled_codec:segment_hash(K), - null}}, + K = + { + o, + list_to_binary(lists:concat(["Bucket", rand:uniform(1024)])), + list_to_binary(lists:concat(["Key", rand:uniform(1024)])), + null + }, + RandKey = + { + K, + {SQN, {active, infinity}, leveled_codec:segment_hash(K), null} + }, generate_randomkeys(Count - 1, SQN + 1, [RandKey|Acc]). clean_testdir(RootPath) -> @@ -1916,12 +2003,13 @@ clean_subdir(DirPath) -> case filelib:is_dir(DirPath) of true -> {ok, Files} = file:list_dir(DirPath), - lists:foreach(fun(FN) -> - File = filename:join(DirPath, FN), - ok = file:delete(File), - io:format("Success deleting ~s~n", [File]) - end, - Files); + lists:foreach( + fun(FN) -> + File = filename:join(DirPath, FN), + ok = file:delete(File), + io:format("Success deleting ~s~n", [File]) + end, + Files); false -> ok end. @@ -1929,15 +2017,18 @@ clean_subdir(DirPath) -> maybe_pause_push(PCL, KL) -> T0 = [], I0 = leveled_pmem:new_index(), - T1 = lists:foldl(fun({K, V}, {AccSL, AccIdx, MinSQN, MaxSQN}) -> - UpdSL = [{K, V}|AccSL], - SQN = leveled_codec:strip_to_seqonly({K, V}), - H = leveled_codec:segment_hash(K), - UpdIdx = leveled_pmem:prepare_for_index(AccIdx, H), - {UpdSL, UpdIdx, min(SQN, MinSQN), max(SQN, MaxSQN)} - end, - {T0, I0, infinity, 0}, - KL), + T1 = + lists:foldl( + fun({K, V}, {AccSL, AccIdx, MinSQN, MaxSQN}) -> + UpdSL = [{K, V}|AccSL], + SQN = leveled_codec:strip_to_seqonly({K, V}), + H = leveled_codec:segment_hash(K), + UpdIdx = leveled_pmem:prepare_for_index(AccIdx, H), + {UpdSL, UpdIdx, min(SQN, MinSQN), max(SQN, MaxSQN)} + end, + {T0, I0, infinity, 0}, + KL + ), SL = element(1, T1), Tree = leveled_tree:from_orderedlist(lists:ukeysort(1, SL), ?CACHE_TYPE), T2 = setelement(1, T1, Tree), @@ -1985,7 +2076,7 @@ shutdown_when_compact(Pid) -> io:format("No outstanding compaction work for ~w~n", [Pid]), pcl_close(Pid). -format_status_test() -> +fetch_status_test() -> RootPath = "test/test_area/ledger", clean_testdir(RootPath), {ok, PCL} = @@ -2038,70 +2129,134 @@ simple_server_test() -> RootPath = "test/test_area/ledger", clean_testdir(RootPath), {ok, PCL} = - pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000, - sst_options=#sst_options{}}), - Key1_Pre = {{o,"Bucket0001", "Key0001", null}, - {1, {active, infinity}, null}}, + pcl_start( + #penciller_options{ + root_path=RootPath, + max_inmemory_tablesize=1000, + sst_options=#sst_options{} + } + ), + Key1_Pre = + { + {o, <<"Bucket0001">>, <<"Key0001">>, null}, + {1, {active, infinity}, null} + }, Key1 = add_missing_hash(Key1_Pre), KL1 = generate_randomkeys({1000, 2}), - Key2_Pre = {{o,"Bucket0002", "Key0002", null}, - {1002, {active, infinity}, null}}, + Key2_Pre = + { + {o, <<"Bucket0002">>, <<"Key0002">>, null}, + {1002, {active, infinity}, null} + }, Key2 = add_missing_hash(Key2_Pre), KL2 = generate_randomkeys({900, 1003}), % Keep below the max table size by having 900 not 1000 - Key3_Pre = {{o,"Bucket0003", "Key0003", null}, - {2003, {active, infinity}, null}}, + Key3_Pre = + { + {o, <<"Bucket0003">>, <<"Key0003">>, null}, + {2003, {active, infinity}, null} + }, Key3 = add_missing_hash(Key3_Pre), KL3 = generate_randomkeys({1000, 2004}), - Key4_Pre = {{o,"Bucket0004", "Key0004", null}, - {3004, {active, infinity}, null}}, + Key4_Pre = + { + {o, <<"Bucket0004">>, <<"Key0004">>, null}, + {3004, {active, infinity}, null} + }, Key4 = add_missing_hash(Key4_Pre), KL4 = generate_randomkeys({1000, 3005}), ok = maybe_pause_push(PCL, [Key1]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), ok = maybe_pause_push(PCL, KL1), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), ok = maybe_pause_push(PCL, [Key2]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), ok = maybe_pause_push(PCL, KL2), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), ok = maybe_pause_push(PCL, [Key3]), - ?assertMatch(Key1, pcl_fetch(PCL, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCL, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCL, {o,"Bucket0003", "Key0003", null})), + ?assertMatch( + Key1, + pcl_fetch(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}) + ), + ?assertMatch( + Key3, + pcl_fetch(PCL, {o, <<"Bucket0003">>, <<"Key0003">>, null})), - true = pcl_checkbloomtest(PCL, {o,"Bucket0001", "Key0001", null}), - true = pcl_checkbloomtest(PCL, {o,"Bucket0002", "Key0002", null}), - true = pcl_checkbloomtest(PCL, {o,"Bucket0003", "Key0003", null}), - false = pcl_checkbloomtest(PCL, {o,"Bucket9999", "Key9999", null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0001">>, <<"Key0001">>, null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0002">>, <<"Key0002">>, null}), + true = pcl_checkbloomtest(PCL, {o, <<"Bucket0003">>, <<"Key0003">>, null}), + false = + pcl_checkbloomtest(PCL, {o, <<"Bucket9999">>, <<"Key9999">>, null}), ok = shutdown_when_compact(PCL), {ok, PCLr} = - pcl_start(#penciller_options{root_path=RootPath, - max_inmemory_tablesize=1000, - sst_options=#sst_options{}}), + pcl_start( + #penciller_options{ + root_path=RootPath, + max_inmemory_tablesize=1000, + sst_options=#sst_options{} + }), ?assertMatch(2003, pcl_getstartupsequencenumber(PCLr)), - % ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0001", "Key0001", null}), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0002", "Key0002", null}), - true = pcl_checkbloomtest(PCLr, {o,"Bucket0003", "Key0003", null}), - false = pcl_checkbloomtest(PCLr, {o,"Bucket9999", "Key9999", null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null}), + true = + pcl_checkbloomtest(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null}), + false = + pcl_checkbloomtest(PCLr, {o, <<"Bucket9999">>, <<"Key9999">>, null}), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), + ?assertMatch( + Key1, + pcl_fetch(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null})), ok = maybe_pause_push(PCLr, KL3), ok = maybe_pause_push(PCLr, [Key4]), ok = maybe_pause_push(PCLr, KL4), - ?assertMatch(Key1, pcl_fetch(PCLr, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PCLr, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PCLr, {o,"Bucket0003", "Key0003", null})), - ?assertMatch(Key4, pcl_fetch(PCLr, {o,"Bucket0004", "Key0004", null})), + + ?assertMatch( + Key1, + pcl_fetch(PCLr, {o, <<"Bucket0001">>, <<"Key0001">>, null}) + ), + ?assertMatch( + Key2, + pcl_fetch(PCLr, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PCLr, {o, <<"Bucket0003">>, <<"Key0003">>, null})), + ?assertMatch( + Key4, + pcl_fetch(PCLr, {o, <<"Bucket0004">>, <<"Key0004">>, null}) + ), {ok, PclSnap, null} = leveled_bookie:snapshot_store( @@ -2113,33 +2268,44 @@ simple_server_test() -> undefined, false), - ?assertMatch(Key1, pcl_fetch(PclSnap, {o,"Bucket0001", "Key0001", null})), - ?assertMatch(Key2, pcl_fetch(PclSnap, {o,"Bucket0002", "Key0002", null})), - ?assertMatch(Key3, pcl_fetch(PclSnap, {o,"Bucket0003", "Key0003", null})), - ?assertMatch(Key4, pcl_fetch(PclSnap, {o,"Bucket0004", "Key0004", null})), + ?assertMatch( + Key1, + pcl_fetch(PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null})), + ?assertMatch( + Key2, + pcl_fetch(PclSnap, {o, <<"Bucket0002">>, <<"Key0002">>, null})), + ?assertMatch( + Key3, + pcl_fetch(PclSnap, {o, <<"Bucket0003">>, <<"Key0003">>, null})), + ?assertMatch( + Key4, + pcl_fetch(PclSnap, {o, <<"Bucket0004">>, <<"Key0004">>, null})), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0002", "Key0002", null}, 1002)), + PclSnap, {o, <<"Bucket0002">>, <<"Key0002">>, null}, 1002)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0003", "Key0003", null}, 2003)), + PclSnap, {o, <<"Bucket0003">>, <<"Key0003">>, null}, 2003)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0004", "Key0004", null}, 3004)), + PclSnap, {o, <<"Bucket0004">>, <<"Key0004">>, null}, 3004)), % Add some more keys and confirm that check sequence number still % sees the old version in the previous snapshot, but will see the new % version in a new snapshot - Key1A_Pre = {{o,"Bucket0001", "Key0001", null}, - {4005, {active, infinity}, null}}, + Key1A_Pre = + { + {o, <<"Bucket0001">>, <<"Key0001">>, null}, + {4005, {active, infinity}, null} + }, Key1A = add_missing_hash(Key1A_Pre), KL1A = generate_randomkeys({2000, 4006}), ok = maybe_pause_push(PCLr, [Key1A]), @@ -2147,7 +2313,7 @@ simple_server_test() -> ?assertMatch( current, pcl_checksequencenumber( - PclSnap, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ok = pcl_close(PclSnap), {ok, PclSnap2, null} = @@ -2163,177 +2329,393 @@ simple_server_test() -> ?assertMatch( replaced, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0001", "Key0001", null}, 1)), + PclSnap2, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 1)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0001", "Key0001", null}, 4005)), + PclSnap2, {o, <<"Bucket0001">>, <<"Key0001">>, null}, 4005)), ?assertMatch( current, pcl_checksequencenumber( - PclSnap2, {o, "Bucket0002", "Key0002", null}, 1002)), + PclSnap2, {o, <<"Bucket0002">>, <<"Key0002">>, null}, 1002)), ok = pcl_close(PclSnap2), ok = pcl_close(PCLr), clean_testdir(RootPath). simple_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {4, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key2", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key2">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key2", null}, - {2, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV3), - {Array5, KV4} = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {4, {active, infinity}, {0, 0}, null}}, - KV4), - ER = find_nextkey(Array5, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key2">>, null}, + {2, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV3), + {Array5, KV4} = + find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + }, + KV4), + ER = + find_nextkey( + Array5, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). sqnoverlap_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {4, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {4, {active, infinity}, {0, 0}, null}}, - KV3), - ER = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {4, {active, infinity}, {0, 0}, null} + }, + KV3), + ER = + find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). sqnoverlap_otherway_findnextkey_test() -> - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, {5, {active, infinity}, {0, 0}, null}}, - {{o, "Bucket1", "Key5", null}, {1, {active, infinity}, {0, 0}, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, {3, {active, infinity}, {0, 0}, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, {2, {active, infinity}, {0, 0}, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {1, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - {Array2, KV1} = find_nextkey(QueryArray, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, {0, 0}, null}}, - KV1), - {Array3, KV2} = find_nextkey(Array2, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, {0, 0}, null}}, - KV2), - {Array4, KV3} = find_nextkey(Array3, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), - ?assertMatch({{o, "Bucket1", "Key5", null}, - {2, {active, infinity}, {0, 0}, null}}, - KV3), - ER = find_nextkey(Array4, - {o, "Bucket1", "Key0", null}, - {o, "Bucket1", "Key5", null}), + {Array2, KV1} = + find_nextkey( + QueryArray, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + KV1), + {Array3, KV2} = + find_nextkey( + Array2, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + }, + KV2), + {Array4, KV3} = + find_nextkey( + Array3, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), + ?assertMatch( + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + }, + KV3), + ER = find_nextkey( + Array4, + {o, <<"Bucket1">>, <<"Key0">>, null}, + {o, <<"Bucket1">>, <<"Key5">>, null} + ), ?assertMatch(no_more_keys, ER). foldwithimm_simple_test() -> Now = leveled_util:integer_now(), - QueryArrayAsList = [ - {2, [{{o, "Bucket1", "Key1", null}, - {5, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key5", null}, - {1, {active, infinity}, 0, null}}]}, - {3, [{{o, "Bucket1", "Key3", null}, - {3, {active, infinity}, 0, null}}]}, - {5, [{{o, "Bucket1", "Key5", null}, - {2, {active, infinity}, 0, null}}]} - ], + QueryArrayAsList = + [ + {2, + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {5, {active, infinity}, {0, 0}, null} + }, + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {1, {active, infinity}, {0, 0}, null} + } + ] + }, + {3, + [ + { + {o, <<"Bucket1">>, <<"Key3">>, null}, + {3, {active, infinity}, {0, 0}, null} + } + ] + }, + {5, + [ + { + {o, <<"Bucket1">>, <<"Key5">>, null}, + {2, {active, infinity}, {0, 0}, null} + } + ] + } + ], QueryArray = convert_qmanifest_tomap(QueryArrayAsList), - KL1A = [{{o, "Bucket1", "Key6", null}, {7, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key1", null}, {8, {active, infinity}, 0, null}}, - {{o, "Bucket1", "Key8", null}, {9, {active, infinity}, 0, null}}], + KL1A = + [ + { + {o, <<"Bucket1">>, <<"Key6">>, null}, + {7, {active, infinity}, 0, null} + }, + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {8, {active, infinity}, 0, null} + }, + { + {o, <<"Bucket1">>, <<"Key8">>, null}, + {9, {active, infinity}, 0, null} + } + ], IMM2 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1A), ?CACHE_TYPE), - IMMiter = leveled_tree:match_range({o, "Bucket1", "Key1", null}, - {o, null, null, null}, - IMM2), - AccFun = fun(K, V, Acc) -> SQN = leveled_codec:strip_to_seqonly({K, V}), - Acc ++ [{K, SQN}] end, - Acc = keyfolder_test(IMMiter, - QueryArray, - {o, "Bucket1", "Key1", null}, {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key5", null}, 2}, - {{o, "Bucket1", "Key6", null}, 7}], Acc), + IMMiter = + leveled_tree:match_range( + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, null, null, null}, + IMM2), + AccFun = + fun(K, V, Acc) -> + SQN = leveled_codec:strip_to_seqonly({K, V}), + Acc ++ [{K, SQN}] + end, + Acc = + keyfolder_test( + IMMiter, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2}, + {{o, <<"Bucket1">>, <<"Key6">>, null}, 7} + ], + Acc), - IMMiterA = [{{o, "Bucket1", "Key1", null}, - {8, {active, infinity}, 0, null}}], - AccA = keyfolder_test(IMMiterA, - QueryArray, - {o, "Bucket1", "Key1", null}, - {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key5", null}, 2}], AccA), + IMMiterA = + [ + { + {o, <<"Bucket1">>, <<"Key1">>, null}, + {8, {active, infinity}, 0, null} + } + ], + AccA = + keyfolder_test( + IMMiterA, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2} + ], + AccA), - AddKV = {{o, "Bucket1", "Key4", null}, {10, {active, infinity}, 0, null}}, + AddKV = + { + {o, <<"Bucket1">>, <<"Key4">>, null}, + {10, {active, infinity}, 0, null} + }, KL1B = [AddKV|KL1A], IMM3 = leveled_tree:from_orderedlist(lists:ukeysort(1, KL1B), ?CACHE_TYPE), - IMMiterB = leveled_tree:match_range({o, "Bucket1", "Key1", null}, - {o, null, null, null}, - IMM3), + IMMiterB = + leveled_tree:match_range( + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, null, null, null}, + IMM3 + ), io:format("Compare IMM3 with QueryArrary~n"), - AccB = keyfolder_test(IMMiterB, - QueryArray, - {o, "Bucket1", "Key1", null}, {o, "Bucket1", "Key6", null}, - {AccFun, [], Now}), - ?assertMatch([{{o, "Bucket1", "Key1", null}, 8}, - {{o, "Bucket1", "Key3", null}, 3}, - {{o, "Bucket1", "Key4", null}, 10}, - {{o, "Bucket1", "Key5", null}, 2}, - {{o, "Bucket1", "Key6", null}, 7}], AccB). + AccB = + keyfolder_test( + IMMiterB, + QueryArray, + {o, <<"Bucket1">>, <<"Key1">>, null}, + {o, <<"Bucket1">>, <<"Key6">>, null}, + {AccFun, [], Now} + ), + ?assertMatch( + [ + {{o, <<"Bucket1">>, <<"Key1">>, null}, 8}, + {{o, <<"Bucket1">>, <<"Key3">>, null}, 3}, + {{o, <<"Bucket1">>, <<"Key4">>, null}, 10}, + {{o, <<"Bucket1">>, <<"Key5">>, null}, 2}, + {{o, <<"Bucket1">>, <<"Key6">>, null}, 7} + ], + AccB). create_file_test() -> {RP, Filename} = {"test/test_area/", "new_file.sst"}, diff --git a/src/leveled_pmanifest.erl b/src/leveled_pmanifest.erl index 35f437a2..63d50389 100644 --- a/src/leveled_pmanifest.erl +++ b/src/leveled_pmanifest.erl @@ -7,7 +7,7 @@ %% each level. This is fine for short-lived volume tests, but as the deeper %% levels are used there will be an exponential penalty. %% -%% The originial intention was to swap out this implementation for a +%% The original intention was to swap out this implementation for a %% multi-version ETS table - but that became complex. So one of two changes %% are pending: %% - Use a single version ES cache for lower levels (and not allow snapshots to @@ -16,6 +16,10 @@ -module(leveled_pmanifest). +% Test uses extracted/edited array related to specific issue, which is not in +% an expected format +-eqwalizer({nowarn_function, potential_issue_test/0}). + -include("leveled.hrl"). -export([ @@ -49,6 +53,17 @@ get_sstpids/1 ]). +-export( + [ + new_entry/5, + entry_startkey/1, + entry_endkey/1, + entry_filename/1, + entry_owner/1, + is_entry/1 + ] +). + -export([ filepath/2 ]). @@ -85,28 +100,41 @@ -define(MANIFESTS_TO_RETAIN, 5). -define(GROOM_SAMPLE, 16). --record(manifest, {levels, - % an array of lists or trees representing the manifest - manifest_sqn = 0 :: non_neg_integer(), - % The current manifest SQN - snapshots = [] - :: list(snapshot()), - % A list of snaphots (i.e. clones) - min_snapshot_sqn = 0 :: integer(), - % The smallest snapshot manifest SQN in the snapshot - % list - pending_deletes = dict:new() :: dict:dict(), - basement :: non_neg_integer(), - % Currently the lowest level (the largest number) - blooms :: dict:dict() - }). +-record(manifest, + { + levels :: array:array(dynamic()), + % an array of lists or trees representing the manifest, where the + % list is created using the to_list function on leveled_treee + manifest_sqn = 0 :: non_neg_integer(), + % The current manifest SQN + snapshots = [] :: list(snapshot()), + % A list of snaphots (i.e. clones) + min_snapshot_sqn = 0 :: integer(), + % The smallest snapshot manifest SQN in the snapshot list + pending_deletes = new_pending_deletions() :: pending_deletions(), + basement :: non_neg_integer(), + % Currently the lowest level (the largest number) + blooms = new_blooms() :: blooms() + }). + +-record(manifest_entry, + { + start_key :: leveled_codec:object_key(), + end_key :: leveled_codec:object_key(), + owner :: pid(), + filename :: string(), + bloom = none :: leveled_ebloom:bloom() | none + } +). -type snapshot() :: {pid(), non_neg_integer(), pos_integer(), pos_integer()}. -type manifest() :: #manifest{}. -type manifest_entry() :: #manifest_entry{}. --type manifest_owner() :: pid()|list(). +-type manifest_owner() :: pid(). -type lsm_level() :: 0..7. +-type pending_deletions() :: dict:dict(). +-type blooms() :: dict:dict(). -type selector_strategy() :: random|{grooming, fun((list(manifest_entry())) -> manifest_entry())}. @@ -128,16 +156,15 @@ new_manifest() -> fun(IDX, Acc) -> array:set(IDX, leveled_tree:empty(?TREE_TYPE), Acc) end, - LevelArray1 = lists:foldl(SetLowerLevelFun, - LevelArray0, - lists:seq(2, ?MAX_LEVELS)), + LevelArray1 = + lists:foldl( + SetLowerLevelFun, LevelArray0, lists:seq(2, ?MAX_LEVELS) + ), #manifest{ levels = LevelArray1, manifest_sqn = 0, snapshots = [], - pending_deletes = dict:new(), - basement = 0, - blooms = dict:new() + basement = 0 }. -spec open_manifest(string()) -> manifest(). @@ -159,9 +186,8 @@ open_manifest(RootPath) -> Acc ++ [list_to_integer(Int)] end end, - ValidManSQNs = lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, - [], - Filenames))), + ValidManSQNs = + lists:reverse(lists:sort(lists:foldl(ExtractSQNFun, [], Filenames))), open_manifestfile(RootPath, ValidManSQNs). -spec copy_manifest(manifest()) -> manifest(). @@ -171,7 +197,9 @@ open_manifest(RootPath) -> copy_manifest(Manifest) -> % Copy the manifest ensuring anything only the master process should care % about is switched to be empty - Manifest#manifest{snapshots = [], pending_deletes = dict:new()}. + Manifest#manifest{ + snapshots = [], pending_deletes = new_pending_deletions() + }. -spec load_manifest( manifest(), @@ -236,10 +264,15 @@ close_manifest(Manifest, CloseEntryFun) -> save_manifest(Manifest, RootPath) -> TFP = filepath(RootPath, Manifest#manifest.manifest_sqn, pending_manifest), AFP = filepath(RootPath, Manifest#manifest.manifest_sqn, current_manifest), - ManBin = term_to_binary(Manifest#manifest{snapshots = [], - pending_deletes = dict:new(), - min_snapshot_sqn = 0, - blooms = dict:new()}), + ManBin = + term_to_binary( + Manifest#manifest{ + snapshots = [], + pending_deletes = new_pending_deletions(), + min_snapshot_sqn = 0, + blooms = new_blooms() + } + ), CRC = erlang:crc32(ManBin), ToPersist = <>, ok = leveled_util:safe_rename(TFP, AFP, ToPersist, true), @@ -324,10 +357,12 @@ report_manifest_level(Manifest, LevelIdx) -> TotalBVBS div LevelSize} end. - --spec replace_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry(), - list()|manifest_entry()) -> manifest(). +-spec replace_manifest_entry( + manifest(), + integer(), + integer(), + list()|manifest_entry(), + list()|manifest_entry()) -> manifest(). %% @doc %% Replace a list of manifest entries in the manifest with a new set of entries %% Pass in the new manifest SQN to be used for this manifest. The list of @@ -342,28 +377,31 @@ replace_manifest_entry(Manifest, ManSQN, LevelIdx, Removals, Additions) -> UpdLevel = replace_entry(LevelIdx, Level, Removals, StrippedAdditions), leveled_log:log(pc019, ["insert", LevelIdx, UpdLevel]), PendingDeletes = - update_pendingdeletes(ManSQN, - Removals, - Manifest#manifest.pending_deletes), + update_pendingdeletes( + ManSQN, Removals, Manifest#manifest.pending_deletes), UpdLevels = array:set(LevelIdx, UpdLevel, Levels), case is_empty(LevelIdx, UpdLevel) of true -> - Manifest#manifest{levels = UpdLevels, - basement = get_basement(UpdLevels), - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms}; + Manifest#manifest{ + levels = UpdLevels, + basement = get_basement(UpdLevels), + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + }; false -> Basement = max(LevelIdx, Manifest#manifest.basement), - Manifest#manifest{levels = UpdLevels, - basement = Basement, - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms} + Manifest#manifest{ + levels = UpdLevels, + basement = Basement, + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + } end. --spec insert_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). +-spec insert_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Place a single new manifest entry into a level of the manifest, at a given %% level and manifest sequence number @@ -375,13 +413,15 @@ insert_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> UpdLevel = add_entry(LevelIdx, Level, UpdEntry), leveled_log:log(pc019, ["insert", LevelIdx, UpdLevel]), Basement = max(LevelIdx, Manifest#manifest.basement), - Manifest#manifest{levels = array:set(LevelIdx, UpdLevel, Levels), - basement = Basement, - manifest_sqn = ManSQN, - blooms = UpdBlooms}. - --spec remove_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). + Manifest#manifest{ + levels = array:set(LevelIdx, UpdLevel, Levels), + basement = Basement, + manifest_sqn = ManSQN, + blooms = UpdBlooms + }. + +-spec remove_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Remove a manifest entry (as it has been merged into the level below) remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> @@ -391,26 +431,30 @@ remove_manifest_entry(Manifest, ManSQN, LevelIdx, Entry) -> update_blooms(Entry, [], Manifest#manifest.blooms), UpdLevel = remove_entry(LevelIdx, Level, Entry), leveled_log:log(pc019, ["remove", LevelIdx, UpdLevel]), - PendingDeletes = update_pendingdeletes(ManSQN, - Entry, - Manifest#manifest.pending_deletes), + PendingDeletes = + update_pendingdeletes( + ManSQN, Entry, Manifest#manifest.pending_deletes), UpdLevels = array:set(LevelIdx, UpdLevel, Levels), case is_empty(LevelIdx, UpdLevel) of true -> - Manifest#manifest{levels = UpdLevels, - basement = get_basement(UpdLevels), - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms}; + Manifest#manifest{ + levels = UpdLevels, + basement = get_basement(UpdLevels), + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + }; false -> - Manifest#manifest{levels = UpdLevels, - manifest_sqn = ManSQN, - pending_deletes = PendingDeletes, - blooms = UpdBlooms} + Manifest#manifest{ + levels = UpdLevels, + manifest_sqn = ManSQN, + pending_deletes = PendingDeletes, + blooms = UpdBlooms + } end. --spec switch_manifest_entry(manifest(), integer(), integer(), - list()|manifest_entry()) -> manifest(). +-spec switch_manifest_entry( + manifest(), integer(), integer(), list()|manifest_entry()) -> manifest(). %% @doc %% Switch a manifest etry from this level to the level below (i.e when there %% are no overlapping manifest entries in the level below) @@ -421,10 +465,8 @@ switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> Level = array:get(SrcLevel, Levels), UpdLevel = remove_entry(SrcLevel, Level, Entry), UpdLevels = array:set(SrcLevel, UpdLevel, Levels), - insert_manifest_entry(Manifest#manifest{levels = UpdLevels}, - ManSQN, - SrcLevel + 1, - Entry). + insert_manifest_entry( + Manifest#manifest{levels = UpdLevels}, ManSQN, SrcLevel + 1, Entry). -spec get_manifest_sqn(manifest()) -> integer(). %% @doc @@ -432,8 +474,9 @@ switch_manifest_entry(Manifest, ManSQN, SrcLevel, Entry) -> get_manifest_sqn(Manifest) -> Manifest#manifest.manifest_sqn. --spec key_lookup(manifest(), integer(), leveled_codec:ledger_key()) - -> false|manifest_owner(). +-spec key_lookup( + manifest(), integer(), leveled_codec:ledger_key()) -> + false|manifest_owner(). %% @doc %% For a given key find which manifest entry covers that key at that level, %% returning false if there is no covering manifest entry at that level. @@ -442,9 +485,8 @@ key_lookup(Manifest, LevelIdx, Key) -> true -> false; false -> - key_lookup_level(LevelIdx, - array:get(LevelIdx, Manifest#manifest.levels), - Key) + key_lookup_level( + LevelIdx, array:get(LevelIdx, Manifest#manifest.levels), Key) end. -spec query_manifest( @@ -514,12 +556,12 @@ merge_lookup(Manifest, LevelIdx, StartKey, EndKey) -> %% Hence, the initial implementation is to select files to merge at random mergefile_selector(Manifest, LevelIdx, _Strategy) when LevelIdx =< 1 -> Level = array:get(LevelIdx, Manifest#manifest.levels), - lists:nth(leveled_rand:uniform(length(Level)), Level); + lists:nth(rand:uniform(length(Level)), Level); mergefile_selector(Manifest, LevelIdx, random) -> Level = leveled_tree:to_list( array:get(LevelIdx, Manifest#manifest.levels)), - {_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level), + {_SK, ME} = lists:nth(rand:uniform(length(Level)), Level), ME; mergefile_selector(Manifest, LevelIdx, {grooming, ScoringFun}) -> Level = @@ -527,7 +569,7 @@ mergefile_selector(Manifest, LevelIdx, {grooming, ScoringFun}) -> array:get(LevelIdx, Manifest#manifest.levels)), SelectorFun = fun(_I, Acc) -> - {_SK, ME} = lists:nth(leveled_rand:uniform(length(Level)), Level), + {_SK, ME} = lists:nth(rand:uniform(length(Level)), Level), [ME|Acc] end, Sample = @@ -549,7 +591,7 @@ merge_snapshot(PencillerManifest, ClerkManifest) -> snapshots = PencillerManifest#manifest.snapshots, min_snapshot_sqn = PencillerManifest#manifest.min_snapshot_sqn}. --spec add_snapshot(manifest(), pid()|atom(), integer()) -> manifest(). +-spec add_snapshot(manifest(), pid(), integer()) -> manifest(). %% @doc %% Add a snapshot reference to the manifest, withe rusing the pid or an atom %% known to reference a special process. The timeout should be in seconds, and @@ -589,9 +631,12 @@ release_snapshot(Manifest, Pid) -> end end end, - {SnapList0, MinSnapSQN, Hit} = lists:foldl(FilterFun, - {[], infinity, false}, - Manifest#manifest.snapshots), + {SnapList0, MinSnapSQN, Hit} = + lists:foldl( + FilterFun, + {[], infinity, false}, + Manifest#manifest.snapshots + ), case Hit of false -> leveled_log:log(p0039, [Pid, length(SnapList0), MinSnapSQN]); @@ -600,12 +645,12 @@ release_snapshot(Manifest, Pid) -> end, case SnapList0 of [] -> - Manifest#manifest{snapshots = SnapList0, - min_snapshot_sqn = 0}; - _ -> + Manifest#manifest{snapshots = SnapList0, min_snapshot_sqn = 0}; + _ when is_integer(MinSnapSQN) -> leveled_log:log(p0004, [SnapList0]), - Manifest#manifest{snapshots = SnapList0, - min_snapshot_sqn = MinSnapSQN} + Manifest#manifest{ + snapshots = SnapList0, min_snapshot_sqn = MinSnapSQN + } end. @@ -733,12 +778,45 @@ get_sstpids(Manifest) -> lists:foldl(FoldFun, [], lists:seq(0, Manifest#manifest.basement)). %%%============================================================================ -%%% Internal Functions +%%% Manifest Entry %%%============================================================================ +-spec new_entry( + leveled_codec:object_key(), + leveled_codec:object_key(), + pid(), + string(), + leveled_ebloom:bloom()|none) -> manifest_entry(). +new_entry(StartKey, EndKey, Owner, FileName, Bloom) -> + #manifest_entry{ + start_key = StartKey, + end_key = EndKey, + owner = Owner, + filename = FileName, + bloom = Bloom + }. + +-spec is_entry(any()) -> boolean(). +is_entry(ME) -> is_record(ME, manifest_entry). + +-spec entry_startkey(manifest_entry()) -> leveled_codec:object_key(). +entry_startkey(ME) -> ME#manifest_entry.start_key. + +-spec entry_endkey(manifest_entry()) -> leveled_codec:object_key(). +entry_endkey(ME) -> ME#manifest_entry.end_key. + +-spec entry_owner(manifest_entry()) -> pid(). +entry_owner(ME) -> ME#manifest_entry.owner. + +-spec entry_filename(manifest_entry()) -> string(). +entry_filename(#manifest_entry{filename = FN}) when ?IS_DEF(FN)-> FN. + +%%%============================================================================ +%%% Internal Functions +%%%============================================================================ --spec get_manifest_entry({tuple(), manifest_entry()}|manifest_entry()) - -> manifest_entry(). +-spec get_manifest_entry( + {tuple(), manifest_entry()}|manifest_entry()) -> manifest_entry(). %% @doc %% Manifest levels can have entries of two forms, use this if only interested %% in the latter form @@ -778,10 +856,12 @@ load_level(LevelIdx, Level, LoadFun, SQNFun) -> lists:foldr(HigherLevelLoadFun, {[], 0, [], []}, Level); false -> {L0, MaxSQN, Flist, UpdBloomL} = - lists:foldr(LowerLevelLoadFun, - {[], 0, [], []}, - leveled_tree:to_list(Level)), - {leveled_tree:from_orderedlist(L0, ?TREE_TYPE, ?TREE_WIDTH), + lists:foldr( + LowerLevelLoadFun, + {[], 0, [], []}, + leveled_tree:to_list(Level) + ), + {leveled_tree:from_orderedlist(L0, ?TREE_TYPE, ?TREE_WIDTH), MaxSQN, Flist, UpdBloomL} @@ -817,9 +897,12 @@ add_entry(_LevelIdx, Level, []) -> Level; add_entry(LevelIdx, Level, Entries) when is_list(Entries) -> FirstEntry = lists:nth(1, Entries), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), case LevelIdx =< 1 of true -> {LHS, RHS} = lists:splitwith(PredFun, Level), @@ -831,9 +914,11 @@ add_entry(LevelIdx, Level, Entries) when is_list(Entries) -> {ME#manifest_entry.end_key, ME} end, Entries0 = lists:map(MapFun, Entries), - leveled_tree:from_orderedlist(lists:append([LHS, Entries0, RHS]), - ?TREE_TYPE, - ?TREE_WIDTH) + leveled_tree:from_orderedlist( + lists:append([LHS, Entries0, RHS]), + ?TREE_TYPE, + ?TREE_WIDTH + ) end. remove_entry(LevelIdx, Level, Entries) -> @@ -861,24 +946,29 @@ remove_section(LevelIdx, Level, FirstEntry, SectionLength) -> false -> {LHS, RHS} = lists:splitwith(PredFun, leveled_tree:to_list(Level)), Post = lists:nthtail(SectionLength, RHS), - leveled_tree:from_orderedlist(lists:append([LHS, Post]), - ?TREE_TYPE, - ?TREE_WIDTH) + leveled_tree:from_orderedlist( + lists:append([LHS, Post]), ?TREE_TYPE, ?TREE_WIDTH) end. replace_entry(LevelIdx, Level, Removals, Additions) when LevelIdx =< 1 -> {SectionLength, FirstEntry} = measure_removals(Removals), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), {LHS, RHS} = lists:splitwith(PredFun, Level), Post = lists:nthtail(SectionLength, RHS), lists:append([LHS, Additions, Post]); replace_entry(LevelIdx, Level, Removals, Additions) -> {SectionLength, FirstEntry} = measure_removals(Removals), - PredFun = pred_fun(LevelIdx, - FirstEntry#manifest_entry.start_key, - FirstEntry#manifest_entry.end_key), + PredFun = + pred_fun( + LevelIdx, + FirstEntry#manifest_entry.start_key, + FirstEntry#manifest_entry.end_key + ), {LHS, RHS} = lists:splitwith(PredFun, leveled_tree:to_list(Level)), Post = case RHS of @@ -898,9 +988,7 @@ replace_entry(LevelIdx, Level, Removals, Additions) -> update_pendingdeletes(ManSQN, Removals, PendingDeletes) -> DelFun = fun(E, Acc) -> - dict:store(E#manifest_entry.filename, - {ManSQN, E}, - Acc) + dict:store(E#manifest_entry.filename, {ManSQN, E}, Acc) end, Entries = case is_list(Removals) of @@ -911,10 +999,11 @@ update_pendingdeletes(ManSQN, Removals, PendingDeletes) -> end, lists:foldl(DelFun, PendingDeletes, Entries). --spec update_blooms(list()|manifest_entry(), - list()|manifest_entry(), - any()) - -> {any(), list()}. +-spec update_blooms( + list()|manifest_entry(), + list()|manifest_entry(), + blooms()) + -> {blooms(), list()}. %% @doc %% %% The manifest is a Pid-> Bloom mappping for every Pid, and this needs to @@ -984,11 +1073,12 @@ range_lookup_int(Manifest, LevelIdx, StartKey, EndKey, MakePointerFun) -> true -> []; false -> - range_lookup_level(LevelIdx, - array:get(LevelIdx, - Manifest#manifest.levels), - StartKey, - EndKey) + range_lookup_level( + LevelIdx, + array:get(LevelIdx, Manifest#manifest.levels), + StartKey, + EndKey + ) end, lists:map(MakePointerFun, Range). @@ -999,8 +1089,9 @@ range_lookup_level(LevelIdx, Level, QStartKey, QEndKey) when LevelIdx =< 1 -> end, NotAfterFun = fun(M) -> - not leveled_codec:endkey_passed(QEndKey, - M#manifest_entry.start_key) + not + leveled_codec:endkey_passed( + QEndKey, M#manifest_entry.start_key) end, {_Before, MaybeIn} = lists:splitwith(BeforeFun, Level), {In, _After} = lists:splitwith(NotAfterFun, MaybeIn), @@ -1016,7 +1107,6 @@ range_lookup_level(_LevelIdx, Level, QStartKey, QEndKey) -> ME end, lists:map(MapFun, Range). - get_basement(Levels) -> GetBaseFun = @@ -1030,7 +1120,6 @@ get_basement(Levels) -> end, lists:foldl(GetBaseFun, 0, lists:seq(0, ?MAX_LEVELS)). - filepath(RootPath, manifest) -> MFP = RootPath ++ "/" ++ ?MANIFEST_FP ++ "/", filelib:ensure_dir(MFP), @@ -1043,9 +1132,6 @@ filepath(RootPath, NewMSN, pending_manifest) -> filepath(RootPath, manifest) ++ "nonzero_" ++ integer_to_list(NewMSN) ++ "." ++ ?PENDING_FILEX. - - - open_manifestfile(_RootPath, L) when L == [] orelse L == [0] -> leveled_log:log(p0013, []), new_manifest(); @@ -1056,7 +1142,11 @@ open_manifestfile(RootPath, [TopManSQN|Rest]) -> case erlang:crc32(BinaryOfTerm) of CRC -> leveled_log:log(p0012, [TopManSQN]), - binary_to_term(BinaryOfTerm); + Manifest = binary_to_term(BinaryOfTerm), + Manifest#manifest{ + pending_deletes = new_pending_deletions(), + blooms = new_blooms() + }; _ -> leveled_log:log(p0033, [CurrManFile, "crc wonky"]), open_manifestfile(RootPath, Rest) @@ -1066,6 +1156,9 @@ seconds_now() -> {MegaNow, SecNow, _} = os:timestamp(), MegaNow * 1000000 + SecNow. +new_blooms() -> dict:new(). + +new_pending_deletions() -> dict:new(). %%%============================================================================ %%% Test @@ -1079,36 +1172,54 @@ initial_setup() -> initial_setup(single_change). initial_setup(Changes) -> - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, - E4 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld7"}, "K93"}, - filename="Z4", - owner="pid_z4", - bloom=none}, - E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, - end_key={o, "Bucket1", "K78", null}, - filename="Z5", - owner="pid_z5", - bloom=none}, - E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, - end_key={o, "Bucket1", "K996", null}, - filename="Z6", - owner="pid_z6", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, + E4 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K93">>}, + filename="Z4", + owner=list_to_pid("<0.104.0>"), + bloom=none + }, + E5 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K78">>, null}, + filename="Z5", + owner=list_to_pid("<0.105.0>"), + bloom=none + }, + E6 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K81">>, null}, + end_key={o, <<"Bucket1">>, <<"K996">>, null}, + filename="Z6", + owner=list_to_pid("<0.106.0>"), + bloom=none + }, initial_setup(Changes, E1, E2, E3, E4, E5, E6). @@ -1137,42 +1248,63 @@ initial_setup(multi_change, E1, E2, E3, E4, E5, E6) -> changeup_setup(Man6) -> - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, - E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, - owner="pid_y1", - filename="Y1", - bloom=none}, - E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - owner="pid_y2", - filename="Y2", - bloom=none}, - E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, - end_key={o, "Bucket1", "K812", null}, - owner="pid_y3", - filename="Y3", - bloom=none}, - E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, - end_key={o, "Bucket1", "K998", null}, - owner="pid_y4", - filename="Y4", - bloom=none}, + E1_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld4">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K62">>}, + owner=list_to_pid("<0.201.0>"), + filename="Y1", + bloom=none + }, + E2_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K67">>}, + end_key={o, <<"Bucket1">>, <<"K45">>, null}, + owner=list_to_pid("<0.202.0>"), + filename="Y2", + bloom=none + }, + E3_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K47">>, null}, + end_key={o, <<"Bucket1">>, <<"K812">>, null}, + owner=list_to_pid("<0.203.0>"), + filename="Y3", + bloom=none + }, + E4_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K815">>, null}, + end_key={o, <<"Bucket1">>, <<"K998">>, null}, + owner=list_to_pid("<0.204.0>"), + filename="Y4", + bloom=none + }, Man7 = remove_manifest_entry(Man6, 2, 1, E1), Man8 = remove_manifest_entry(Man7, 2, 1, E2), @@ -1211,11 +1343,11 @@ manifest_gc_test() -> keylookup_manifest_test() -> {Man0, Man1, Man2, Man3, _Man4, _Man5, Man6} = initial_setup(), - LK1_1 = {o, "Bucket1", "K711", null}, - LK1_2 = {o, "Bucket1", "K70", null}, - LK1_3 = {o, "Bucket1", "K71", null}, - LK1_4 = {o, "Bucket1", "K75", null}, - LK1_5 = {o, "Bucket1", "K76", null}, + LK1_1 = {o, <<"Bucket1">>, <<"K711">>, null}, + LK1_2 = {o, <<"Bucket1">>, <<"K70">>, null}, + LK1_3 = {o, <<"Bucket1">>, <<"K71">>, null}, + LK1_4 = {o, <<"Bucket1">>, <<"K75">>, null}, + LK1_5 = {o, <<"Bucket1">>, <<"K76">>, null}, ?assertMatch(false, key_lookup(Man0, 1, LK1_1)), ?assertMatch(false, key_lookup(Man1, 1, LK1_1)), @@ -1223,15 +1355,19 @@ keylookup_manifest_test() -> ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + PZ2 = list_to_pid("<0.102.0>"), + PZ3 = list_to_pid("<0.103.0>"), + PZ5 = list_to_pid("<0.105.0>"), + + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_2)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_3)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_4)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_5)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_2)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_3)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_5)), {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, Man13} = changeup_setup(Man6), @@ -1242,18 +1378,20 @@ keylookup_manifest_test() -> ?assertMatch(false, key_lookup(Man3, 1, LK1_1)), ?assertMatch(false, key_lookup(Man6, 1, LK1_1)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_2)), - ?assertMatch("pid_z2", key_lookup(Man6, 1, LK1_3)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_4)), - ?assertMatch("pid_z3", key_lookup(Man6, 1, LK1_5)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_2)), + ?assertMatch(PZ2, key_lookup(Man6, 1, LK1_3)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_4)), + ?assertMatch(PZ3, key_lookup(Man6, 1, LK1_5)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_2)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_3)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man6, 2, LK1_5)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_2)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_3)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man6, 2, LK1_5)), - ?assertMatch("pid_y3", key_lookup(Man13, 1, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man13, 2, LK1_4)). + PY3 = list_to_pid("<0.203.0>"), + + ?assertMatch(PY3, key_lookup(Man13, 1, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man13, 2, LK1_4)). ext_keylookup_manifest_test() -> RP = "test/test_area", @@ -1261,10 +1399,13 @@ ext_keylookup_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), save_manifest(Man6, RP), - E7 = #manifest_entry{start_key={o, "Bucket1", "K997", null}, - end_key={o, "Bucket1", "K999", null}, - filename="Z7", - owner="pid_z7"}, + E7 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K997">>, null}, + end_key={o, <<"Bucket1">>, <<"K999">>, null}, + filename="Z7", + owner=list_to_pid("<0.107.0>") + }, Man7 = insert_manifest_entry(Man6, 2, 2, E7), save_manifest(Man7, RP), ManOpen1 = open_manifest(RP), @@ -1275,7 +1416,7 @@ ext_keylookup_manifest_test() -> {ok, BytesCopied} = file:copy(Man7FN, Man7FNAlt), {ok, Bin} = file:read_file(Man7FN), ?assertMatch(BytesCopied, byte_size(Bin)), - RandPos = leveled_rand:uniform(bit_size(Bin) - 1), + RandPos = rand:uniform(bit_size(Bin) - 1), <> = Bin, Flipped = BitToFlip bxor 1, ok = file:write_file(Man7FN, @@ -1288,62 +1429,93 @@ ext_keylookup_manifest_test() -> ManOpen2 = open_manifest(RP), ?assertMatch(1, get_manifest_sqn(ManOpen2)), - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, - E1_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld4"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K62"}, - owner="pid_y1", - filename="Y1", - bloom=none}, - E2_2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K67"}, - end_key={o, "Bucket1", "K45", null}, - owner="pid_y2", - filename="Y2", - bloom=none}, - E3_2 = #manifest_entry{start_key={o, "Bucket1", "K47", null}, - end_key={o, "Bucket1", "K812", null}, - owner="pid_y3", - filename="Y3", - bloom=none}, - E4_2 = #manifest_entry{start_key={o, "Bucket1", "K815", null}, - end_key={o, "Bucket1", "K998", null}, - owner="pid_y4", - filename="Y4", - bloom=none}, + E1_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld4">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K62">>}, + owner=list_to_pid("<0.201.0>"), + filename="Y1", + bloom=none + }, + E2_2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K67">>}, + end_key={o, <<"Bucket1">>, <<"K45">>, null}, + owner=list_to_pid("<0.202.0>"), + filename="Y2", + bloom=none + }, + E3_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K47">>, null}, + end_key={o, <<"Bucket1">>, <<"K812">>, null}, + owner=list_to_pid("<0.203.0>"), + filename="Y3", + bloom=none + }, + E4_2 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K815">>, null}, + end_key={o, <<"Bucket1">>, <<"K998">>, null}, + owner=list_to_pid("<0.104.0>"), + filename="Y4", + bloom=none + }, Man8 = replace_manifest_entry(ManOpen2, 2, 1, E1, E1_2), Man9 = remove_manifest_entry(Man8, 2, 1, [E2, E3]), Man10 = insert_manifest_entry(Man9, 2, 1, [E2_2, E3_2, E4_2]), ?assertMatch(2, get_manifest_sqn(Man10)), - LK1_4 = {o, "Bucket1", "K75", null}, - ?assertMatch("pid_y3", key_lookup(Man10, 1, LK1_4)), - ?assertMatch("pid_z5", key_lookup(Man10, 2, LK1_4)), + LK1_4 = {o, <<"Bucket1">>, <<"K75">>, null}, + + PY3 = list_to_pid("<0.203.0>"), + PZ5 = list_to_pid("<0.105.0>"), + + ?assertMatch(PY3, key_lookup(Man10, 1, LK1_4)), + ?assertMatch(PZ5, key_lookup(Man10, 2, LK1_4)), - E5 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld7"}, "K97"}, - end_key={o, "Bucket1", "K78", null}, - filename="Z5", - owner="pid_z5", - bloom=none}, - E6 = #manifest_entry{start_key={o, "Bucket1", "K81", null}, - end_key={o, "Bucket1", "K996", null}, - filename="Z6", - owner="pid_z6", - bloom=none}, + E5 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld7">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K78">>, null}, + filename="Z5", + owner=PZ5, + bloom=none + }, + E6 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K81">>, null}, + end_key={o, <<"Bucket1">>, <<"K996">>, null}, + filename="Z6", + owner=list_to_pid("<0.106.0>"), + bloom=none + }, Man11 = remove_manifest_entry(Man10, 3, 2, [E5, E6]), ?assertMatch(3, get_manifest_sqn(Man11)), @@ -1351,7 +1523,7 @@ ext_keylookup_manifest_test() -> Man12 = replace_manifest_entry(Man11, 4, 2, E2_2, E5), ?assertMatch(4, get_manifest_sqn(Man12)), - ?assertMatch("pid_z5", key_lookup(Man12, 2, LK1_4)). + ?assertMatch(PZ5, key_lookup(Man12, 2, LK1_4)). rangequery_manifest_test() -> {_Man0, _Man1, _Man2, _Man3, _Man4, _Man5, Man6} = initial_setup(), @@ -1361,50 +1533,60 @@ rangequery_manifest_test() -> {next, ME, _SK} = Pointer, ME#manifest_entry.owner end, + + PZ1 = list_to_pid("<0.101.0>"), + PZ3 = list_to_pid("<0.103.0>"), + PZ5 = list_to_pid("<0.105.0>"), + PZ6 = list_to_pid("<0.106.0>"), + PY1 = list_to_pid("<0.201.0>"), + PY3 = list_to_pid("<0.203.0>"), + PY4 = list_to_pid("<0.204.0>"), - SK1 = {o, "Bucket1", "K711", null}, - EK1 = {o, "Bucket1", "K999", null}, + SK1 = {o, <<"Bucket1">>, <<"K711">>, null}, + EK1 = {o, <<"Bucket1">>, <<"K999">>, null}, RL1_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), - ?assertMatch(["pid_z3"], RL1_1), + ?assertMatch([PZ3], RL1_1), RL1_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK1, EK1)), - ?assertMatch(["pid_z5", "pid_z6"], RL1_2), - SK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, - EK2 = {i, "Bucket1", {"Idx1", "Fld8"}, null}, + ?assertMatch([PZ5, PZ6], RL1_2), + SK2 = {i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld8">>}, null}, + EK2 = {i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld8">>}, null}, RL2_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), - ?assertMatch(["pid_z1"], RL2_1), + ?assertMatch([PZ1], RL2_1), RL2_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK2, EK2)), - ?assertMatch(["pid_z5"], RL2_2), + ?assertMatch([PZ5], RL2_2), - SK3 = {o, "Bucket1", "K994", null}, - EK3 = {o, "Bucket1", "K995", null}, + SK3 = {o, <<"Bucket1">>, <<"K994">>, null}, + EK3 = {o, <<"Bucket1">>, <<"K995">>, null}, RL3_1 = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1), RL3_2 = lists:map(PidMapFun, range_lookup(Man6, 2, SK3, EK3)), - ?assertMatch(["pid_z6"], RL3_2), + ?assertMatch([PZ6], RL3_2), {_Man7, _Man8, _Man9, _Man10, _Man11, _Man12, Man13} = changeup_setup(Man6), RL1_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK1, EK1)), - ?assertMatch(["pid_z3"], RL1_1A), + ?assertMatch([PZ3], RL1_1A), RL2_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK2, EK2)), - ?assertMatch(["pid_z1"], RL2_1A), + ?assertMatch([PZ1], RL2_1A), RL3_1A = lists:map(PidMapFun, range_lookup(Man6, 1, SK3, EK3)), ?assertMatch([], RL3_1A), RL1_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK1, EK1)), - ?assertMatch(["pid_y3", "pid_y4"], RL1_1B), + ?assertMatch([PY3, PY4], RL1_1B), RL2_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK2, EK2)), - ?assertMatch(["pid_y1"], RL2_1B), + ?assertMatch([PY1], RL2_1B), RL3_1B = lists:map(PidMapFun, range_lookup(Man13, 1, SK3, EK3)), - ?assertMatch(["pid_y4"], RL3_1B). + ?assertMatch([PY4], RL3_1B). levelzero_present_test() -> - E0 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={o, "Bucket1", "Key996", null}, - filename="Z0", - owner="pid_z0", - bloom=none}, + E0 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={o, <<"Bucket1">>, <<"Key996">>, null}, + filename="Z0", + owner=list_to_pid("<0.101.0>"), + bloom=none}, Man0 = new_manifest(), ?assertMatch(false, levelzero_present(Man0)), @@ -1426,21 +1608,30 @@ snapshot_release_test() -> PidA3 = spawn(fun() -> ok end), PidA4 = spawn(fun() -> ok end), Man6 = element(7, initial_setup()), - E1 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld1"}, "K8"}, - end_key={i, "Bucket1", {"Idx1", "Fld9"}, "K93"}, - filename="Z1", - owner="pid_z1", - bloom=none}, - E2 = #manifest_entry{start_key={i, "Bucket1", {"Idx1", "Fld9"}, "K97"}, - end_key={o, "Bucket1", "K71", null}, - filename="Z2", - owner="pid_z2", - bloom=none}, - E3 = #manifest_entry{start_key={o, "Bucket1", "K75", null}, - end_key={o, "Bucket1", "K993", null}, - filename="Z3", - owner="pid_z3", - bloom=none}, + E1 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld1">>}, <<"K8">>}, + end_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K93">>}, + filename="Z1", + owner=list_to_pid("<0.101.0>"), + bloom=none + }, + E2 = + #manifest_entry{ + start_key={i, <<"Bucket1">>, {<<"Idx1">>, <<"Fld9">>}, <<"K97">>}, + end_key={o, <<"Bucket1">>, <<"K71">>, null}, + filename="Z2", + owner=list_to_pid("<0.102.0>"), + bloom=none + }, + E3 = + #manifest_entry{ + start_key={o, <<"Bucket1">>, <<"K75">>, null}, + end_key={o, <<"Bucket1">>, <<"K993">>, null}, + filename="Z3", + owner=list_to_pid("<0.103.0>"), + bloom=none + }, Man7 = add_snapshot(Man6, PidA1, 3600), Man8 = remove_manifest_entry(Man7, 2, 1, E1), @@ -1493,39 +1684,79 @@ snapshot_timeout_test() -> potential_issue_test() -> Manifest = - {manifest,{array,9,0,[], - {[], - [{manifest_entry,{o_rkv,"Bucket","Key10",null}, - {o_rkv,"Bucket","Key12949",null}, - "<0.313.0>","./16_1_0.sst", none}, - {manifest_entry,{o_rkv,"Bucket","Key129490",null}, - {o_rkv,"Bucket","Key158981",null}, - "<0.315.0>","./16_1_1.sst", none}, - {manifest_entry,{o_rkv,"Bucket","Key158982",null}, - {o_rkv,"Bucket","Key188472",null}, - "<0.316.0>","./16_1_2.sst", none}], - {idxt,1, - {{[{{o_rkv,"Bucket1","Key1",null}, - {manifest_entry,{o_rkv,"Bucket","Key9083",null}, - {o_rkv,"Bucket1","Key1",null}, - "<0.320.0>","./16_1_6.sst", none}}]}, - {1,{{o_rkv,"Bucket1","Key1",null},1,nil,nil}}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - {idxt,0,{{},{0,nil}}}, - []}}, - 19, [], 0, dict:new(), 2, dict:new()}, - Range1 = range_lookup(Manifest, - 1, - {o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}), - Range2 = range_lookup(Manifest, - 2, - {o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}), + {manifest, + {array,9,0,[], + { + [], + [ + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key10">>, null}, + {o_rkv, <<"Bucket">>, <<"Key12949">>,null}, + list_to_pid("<0.313.0>"), + "./16_1_0.sst", + none + }, + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key129490">>, null}, + {o_rkv, <<"Bucket">>, <<"Key158981">>, null}, + list_to_pid("<0.315.0>"), + "./16_1_1.sst", + none + }, + {manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key158982">>, null}, + {o_rkv, <<"Bucket">>, <<"Key188472">>, null}, + list_to_pid("<0.316.0>"), + "./16_1_2.sst", + none + } + ], + { + idxt, + 1, + { + { + [ + {{o_rkv, <<"Bucket1">>, <<"Key1">>, null}, + { + manifest_entry, + {o_rkv, <<"Bucket">>, <<"Key9083">>, null}, + {o_rkv, <<"Bucket1">>, <<"Key1">>, null}, + list_to_pid("<0.320.0>"), + "./16_1_6.sst", + none + } + } + ] + }, + {1, {{o_rkv, <<"Bucket1">> ,<<"Key1">> ,null},1,nil,nil}}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + {idxt,0,{{},{0,nil}}}, + []}}, + 19, + [], + 0, + new_pending_deletions(), + 2, + new_blooms()}, + Range1 = + range_lookup( + Manifest, + 1, + {o_rkv, <<"Bucket">>, null, null}, + {o_rkv, <<"Bucket">>, null, null} + ), + Range2 = + range_lookup( + Manifest, + 2, + {o_rkv, <<"Bucket">>, null, null}, + {o_rkv, <<"Bucket">>, null, null} + ), io:format("Range in Level 1 ~w~n", [Range1]), io:format("Range in Level 2 ~w~n", [Range2]), ?assertMatch(3, length(Range1)), diff --git a/src/leveled_pmem.erl b/src/leveled_pmem.erl index ce7b9e46..db97a998 100644 --- a/src/leveled_pmem.erl +++ b/src/leveled_pmem.erl @@ -41,9 +41,12 @@ cache_full/1 ]). +% Test functions to ignore for equalizer - due to array issues +-eqwalizer({nowarn_function, index_performance_test/0}). + -define(MAX_CACHE_LINES, 31). % Must be less than 128 --type index_array() :: list(array:array())|[]|none. +-type index_array() :: list(array:array(binary()))|none. -export_type([index_array/0]). @@ -58,7 +61,7 @@ cache_full(L0Cache) -> length(L0Cache) == ?MAX_CACHE_LINES. -spec prepare_for_index( - array:array(), leveled_codec:segment_hash()) -> array:array(). + array:array(binary()), leveled_codec:segment_hash()) -> array:array(). %% @doc %% Add the hash of a key to the index. This is 'prepared' in the sense that %% this index is not use until it is loaded into the main index. @@ -73,18 +76,22 @@ prepare_for_index(IndexArray, Hash) -> Bin = array:get(Slot, IndexArray), array:set(Slot, <>, IndexArray). --spec add_to_index(array:array(), index_array(), integer()) -> index_array(). +-spec add_to_index( + array:array(binary()), index_array(), integer()) -> index_array(). %% @doc %% Expand the penciller's current index array with the details from a new %% ledger cache tree sent from the Bookie. The tree will have a cache slot %% which is the index of this ledger_cache in the list of the ledger_caches -add_to_index(LM1Array, L0Index, CacheSlot) when CacheSlot < 128 -> +add_to_index( + LM1Array, L0Index, CacheSlot) + when CacheSlot < 128, L0Index =/= none -> [LM1Array|L0Index]. --spec new_index() -> array:array(). +-spec new_index() -> array:array(binary()). %% @doc %% Create a new index array new_index() -> + % eqwalizer:ignore - array does contain binary() array:new([{size, 256}, {default, <<>>}]). -spec check_index(leveled_codec:segment_hash(), index_array()) @@ -92,7 +99,7 @@ new_index() -> %% @doc %% return a list of positions in the list of cache arrays that may contain the %% key associated with the hash being checked -check_index(Hash, L0Index) -> +check_index(Hash, L0Index) when L0Index =/= none -> {Slot, H0} = split_hash(Hash), {_L, Positions} = lists:foldl( @@ -239,19 +246,14 @@ check_slotlist(Key, _Hash, CheckList, TreeList) -> -include_lib("eunit/include/eunit.hrl"). generate_randomkeys_aslist(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> - lists:ukeysort(1, - generate_randomkeys(Seqn, - Count, - [], - BucketRangeLow, - BucketRangeHigh)). + lists:ukeysort( + 1, + generate_randomkeys(Seqn, Count, [], BucketRangeLow, BucketRangeHigh) + ). generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> - KVL = generate_randomkeys(Seqn, - Count, - [], - BucketRangeLow, - BucketRangeHigh), + KVL = + generate_randomkeys(Seqn, Count, [], BucketRangeLow, BucketRangeHigh), leveled_tree:from_orderedlist(lists:ukeysort(1, KVL), ?CACHE_TYPE). generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> @@ -260,32 +262,35 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> BNumber = lists:flatten( io_lib:format("~4..0B", - [BucketLow + leveled_rand:uniform(BRange)])), + [BucketLow + rand:uniform(BRange)])), KNumber = - lists:flatten(io_lib:format("~4..0B", [leveled_rand:uniform(1000)])), - {K, V} = {{o, "Bucket" ++ BNumber, "Key" ++ KNumber, null}, - {Seqn, {active, infinity}, null}}, - generate_randomkeys(Seqn + 1, - Count - 1, - [{K, V}|Acc], - BucketLow, - BRange). + lists:flatten(io_lib:format("~4..0B", [rand:uniform(1000)])), + {K, V} = + { + {o, + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + null}, + {Seqn, {active, infinity}, null} + }, + generate_randomkeys(Seqn + 1, Count - 1, [{K, V}|Acc], BucketLow, BRange). compare_method_test() -> - R = lists:foldl(fun(_X, {LedgerSQN, L0Size, L0TreeList}) -> - LM1 = generate_randomkeys(LedgerSQN + 1, - 2000, 1, 500), - add_to_cache( - L0Size, - {LM1, LedgerSQN + 1, LedgerSQN + 2000}, - LedgerSQN, - L0TreeList, - true) - end, - {0, 0, []}, - lists:seq(1, 16)), - + R = + lists:foldl( + fun(_X, {LedgerSQN, L0Size, L0TreeList}) -> + LM1 = generate_randomkeys(LedgerSQN + 1, 2000, 1, 500), + add_to_cache( + L0Size, + {LM1, LedgerSQN + 1, LedgerSQN + 2000}, + LedgerSQN, + L0TreeList, + true) + end, + {0, 0, []}, + lists:seq(1, 16)), + {SQN, Size, TreeList} = R, ?assertMatch(32000, SQN), ?assertMatch(true, Size =< 32000), @@ -310,51 +315,62 @@ compare_method_test() -> end end, - S0 = lists:foldl(fun({Key, _V}, Acc) -> - R0 = lists:foldl(FindKeyFun(Key), - {false, not_found}, - TreeList), - [R0|Acc] end, - [], - TestList), + S0 = + lists:foldl( + fun({Key, _V}, Acc) -> + R0 = + lists:foldl( + FindKeyFun(Key), {false, not_found}, TreeList), + [R0|Acc] + end, + [], + TestList) + , PosList = lists:seq(1, length(TreeList)), - S1 = lists:foldl(fun({Key, _V}, Acc) -> - R0 = check_levelzero(Key, PosList, TreeList), - [R0|Acc] - end, - [], - TestList), + S1 = + lists:foldl( + fun({Key, _V}, Acc) -> + R0 = check_levelzero(Key, PosList, TreeList), + [R0|Acc] + end, + [], + TestList + ), ?assertMatch(S0, S1), - StartKey = {o, "Bucket0100", null, null}, - EndKey = {o, "Bucket0200", null, null}, + StartKey = {o, <<"Bucket0100">>, null, null}, + EndKey = {o, <<"Bucket0200">>, null, null}, SWa = os:timestamp(), FetchFun = fun(Slot) -> lists:nth(Slot, TreeList) end, DumpList = to_list(length(TreeList), FetchFun), - Q0 = lists:foldl(fun({K, V}, Acc) -> - P = leveled_codec:endkey_passed(EndKey, K), - case {K, P} of - {K, false} when K >= StartKey -> - [{K, V}|Acc]; - _ -> - Acc - end - end, - [], - DumpList), + Q0 = + lists:foldl( + fun({K, V}, Acc) -> + P = leveled_codec:endkey_passed(EndKey, K), + case {K, P} of + {K, false} when K >= StartKey -> + [{K, V}|Acc]; + _ -> + Acc + end + end, + [], + DumpList + ), Tree = leveled_tree:from_orderedlist(lists:ukeysort(1, Q0), ?CACHE_TYPE), Sz0 = leveled_tree:tsize(Tree), - io:format("Crude method took ~w microseconds resulting in tree of " ++ - "size ~w~n", - [timer:now_diff(os:timestamp(), SWa), Sz0]), + io:format( + "Crude method took ~w microseconds resulting in tree of size ~w~n", + [timer:now_diff(os:timestamp(), SWa), Sz0] + ), SWb = os:timestamp(), Q1 = merge_trees(StartKey, EndKey, TreeList, leveled_tree:empty(?CACHE_TYPE)), Sz1 = length(Q1), - io:format("Merge method took ~w microseconds resulting in tree of " ++ - "size ~w~n", - [timer:now_diff(os:timestamp(), SWb), Sz1]), + io:format( + "Merge method took ~w microseconds resulting in tree of size ~w~n", + [timer:now_diff(os:timestamp(), SWb), Sz1]), ?assertMatch(Sz0, Sz1). with_index_test_() -> diff --git a/src/leveled_rand.erl b/src/leveled_rand.erl deleted file mode 100644 index fedaf20b..00000000 --- a/src/leveled_rand.erl +++ /dev/null @@ -1,28 +0,0 @@ -%% Generalized random module that offers a backwards compatible API -%% around some of the changes in rand, crypto and for time units. - --module(leveled_rand). - -%% API --export([ - uniform/0, - uniform/1, - seed/0, - rand_bytes/1 - ]). - -%%%=================================================================== -%%% New (r19+) rand style functions -%%%=================================================================== -uniform() -> - rand:uniform(). - -uniform(N) -> - rand:uniform(N). - -seed() -> - ok. - -rand_bytes(Size) -> - crypto:strong_rand_bytes(Size). - diff --git a/src/leveled_runner.erl b/src/leveled_runner.erl index 32fe5f16..9737162f 100644 --- a/src/leveled_runner.erl +++ b/src/leveled_runner.erl @@ -39,8 +39,7 @@ -define(CHECKJOURNAL_PROB, 0.2). -type key_range() - :: {leveled_codec:ledger_key()|null, - leveled_codec:ledger_key()|null}. + :: {leveled_codec:query_key(), leveled_codec:query_key()}. -type foldacc() :: any(). % Can't currently be specific about what an acc might be @@ -56,15 +55,15 @@ :: fun((leveled_codec:key(), leveled_codec:key()) -> accumulate|pass). -type snap_fun() - :: fun(() -> {ok, pid(), pid()|null}). + :: fun(() -> {ok, pid(), pid()|null, fun(() -> ok)}). -type runner_fun() :: fun(() -> foldacc()). --type acc_fun() - :: fun((leveled_codec:key(), any(), foldacc()) -> foldacc()). +-type objectacc_fun() + :: fun((leveled_codec:object_key(), any(), foldacc()) -> foldacc()). -type mp() :: {re_pattern, term(), term(), term(), term()}. --export_type([acc_fun/0, mp/0]). +-export_type([fold_keys_fun/0, mp/0]). %%%============================================================================ %%% External functions @@ -76,8 +75,8 @@ %% @doc %% Fold over a bucket accumulating the count of objects and their total sizes bucket_sizestats(SnapFun, Bucket, Tag) -> - StartKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), - EndKey = leveled_codec:to_ledgerkey(Bucket, null, Tag), + StartKey = leveled_codec:to_querykey(Bucket, null, Tag), + EndKey = leveled_codec:to_querykey(Bucket, null, Tag), AccFun = accumulate_size(), Runner = fun() -> @@ -132,7 +131,7 @@ bucket_list(SnapFun, Tag, FoldBucketsFun, InitAcc, MaxBuckets) -> -spec index_query(snap_fun(), {leveled_codec:ledger_key(), leveled_codec:ledger_key(), - {boolean(), undefined|mp()|iodata()}}, + {boolean(), undefined|mp()}}, {fold_keys_fun(), foldacc()}) -> {async, runner_fun()}. %% @doc @@ -165,7 +164,7 @@ index_query(SnapFun, {StartKey, EndKey, TermHandling}, FoldAccT) -> -spec bucketkey_query(snap_fun(), leveled_codec:tag(), leveled_codec:key()|null, - key_range(), + {leveled_codec:single_key()|null, leveled_codec:single_key()|null}, {fold_keys_fun(), foldacc()}, leveled_codec:regular_expression()) -> {async, runner_fun()}. @@ -175,8 +174,8 @@ bucketkey_query(SnapFun, Tag, Bucket, {StartKey, EndKey}, {FoldKeysFun, InitAcc}, TermRegex) -> - SK = leveled_codec:to_ledgerkey(Bucket, StartKey, Tag), - EK = leveled_codec:to_ledgerkey(Bucket, EndKey, Tag), + SK = leveled_codec:to_querykey(Bucket, StartKey, Tag), + EK = leveled_codec:to_querykey(Bucket, EndKey, Tag), AccFun = accumulate_keys(FoldKeysFun, TermRegex), Runner = fun() -> @@ -203,8 +202,8 @@ bucketkey_query(SnapFun, Tag, Bucket, FunAcc) -> %% @doc %% Fold over the keys under a given Tag accumulating the hashes hashlist_query(SnapFun, Tag, JournalCheck) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), Runner = fun() -> {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = SnapFun(), @@ -217,10 +216,11 @@ hashlist_query(SnapFun, Tag, JournalCheck) -> end, {async, Runner}. --spec tictactree(snap_fun(), - {leveled_codec:tag(), leveled_codec:key(), tuple()}, - boolean(), atom(), fold_filter_fun()) - -> {async, runner_fun()}. +-spec tictactree( + snap_fun(), + {leveled_codec:tag(), leveled_codec:key(), tuple()}, + boolean(), leveled_tictac:tree_size(), fold_filter_fun()) + -> {async, runner_fun()}. %% @doc %% Return a merkle tree from the fold, directly accessing hashes cached in the %% metadata @@ -246,14 +246,14 @@ tictactree(SnapFun, {Tag, Bucket, Query}, JournalCheck, TreeSize, Filter) -> case Tag of ?IDX_TAG -> {IdxFld, StartIdx, EndIdx} = Query, - KeyDefFun = fun leveled_codec:to_ledgerkey/5, + KeyDefFun = fun leveled_codec:to_querykey/5, {KeyDefFun(Bucket, null, ?IDX_TAG, IdxFld, StartIdx), KeyDefFun(Bucket, null, ?IDX_TAG, IdxFld, EndIdx), EnsureKeyBinaryFun}; _ -> {StartOKey, EndOKey} = Query, - {leveled_codec:to_ledgerkey(Bucket, StartOKey, Tag), - leveled_codec:to_ledgerkey(Bucket, EndOKey, Tag), + {leveled_codec:to_querykey(Bucket, StartOKey, Tag), + leveled_codec:to_querykey(Bucket, EndOKey, Tag), fun(K, H) -> V = {is_hash, H}, EnsureKeyBinaryFun(K, V) @@ -279,8 +279,8 @@ tictactree(SnapFun, {Tag, Bucket, Query}, JournalCheck, TreeSize, Filter) -> %% function to each proxy object foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, SegmentList, LastModRange, MaxObjectCount) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -298,8 +298,8 @@ foldheads_allkeys(SnapFun, Tag, FoldFun, JournalCheck, %% @doc %% Fold over all objects for a given tag foldobjects_allkeys(SnapFun, Tag, FoldFun, key_order) -> - StartKey = leveled_codec:to_ledgerkey(null, null, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(null, null, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -335,7 +335,7 @@ foldobjects_allkeys(SnapFun, Tag, FoldObjectsFun, sqn_order) -> _ -> {VBin, _VSize} = ExtractFun(JVal), {Obj, _IdxSpecs} = - leveled_codec:split_inkvalue(VBin), + leveled_codec:revert_value_from_journal(VBin), ToLoop = case SQN of MaxSQN -> stop; @@ -353,11 +353,16 @@ foldobjects_allkeys(SnapFun, Tag, FoldObjectsFun, sqn_order) -> Folder = fun() -> - {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = SnapFun(), - {ok, JournalSQN} = leveled_inker:ink_getjournalsqn(JournalSnapshot), + {ok, LedgerSnapshot, JournalSnapshot, AfterFun} = + case SnapFun() of + {ok, LS, JS, AF} when is_pid(JS) -> + {ok, LS, JS, AF} + end, + {ok, JournalSQN} = + leveled_inker:ink_getjournalsqn(JournalSnapshot), IsValidFun = fun(Bucket, Key, SQN) -> - LedgerKey = leveled_codec:to_ledgerkey(Bucket, Key, Tag), + LedgerKey = leveled_codec:to_objectkey(Bucket, Key, Tag), CheckSQN = leveled_penciller:pcl_checksequencenumber( LedgerSnapshot, LedgerKey, SQN), @@ -438,9 +443,9 @@ foldheads_bybucket(SnapFun, %% and passing those objects into the fold function foldobjects_byindex(SnapFun, {Tag, Bucket, Field, FromTerm, ToTerm}, FoldFun) -> StartKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, FromTerm), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, Field, FromTerm), EndKey = - leveled_codec:to_ledgerkey(Bucket, null, ?IDX_TAG, Field, ToTerm), + leveled_codec:to_querykey(Bucket, null, ?IDX_TAG, Field, ToTerm), foldobjects(SnapFun, Tag, [{StartKey, EndKey}], @@ -457,38 +462,39 @@ foldobjects_byindex(SnapFun, {Tag, Bucket, Field, FromTerm, ToTerm}, FoldFun) -> get_nextbucket(_NextB, _NextK, _Tag, _LS, BKList, {Limit, Limit}) -> lists:reverse(BKList); get_nextbucket(NextBucket, NextKey, Tag, LedgerSnapshot, BKList, {C, L}) -> - StartKey = leveled_codec:to_ledgerkey(NextBucket, NextKey, Tag), - EndKey = leveled_codec:to_ledgerkey(null, null, Tag), + StartKey = leveled_codec:to_querykey(NextBucket, NextKey, Tag), + EndKey = leveled_codec:to_querykey(null, null, Tag), ExtractFun = fun(LK, V, _Acc) -> {leveled_codec:from_ledgerkey(LK), V} end, - R = leveled_penciller:pcl_fetchnextkey(LedgerSnapshot, - StartKey, - EndKey, - ExtractFun, - null), + R = + leveled_penciller:pcl_fetchnextkey( + LedgerSnapshot, StartKey, EndKey, ExtractFun, null), case R of {1, null} -> leveled_log:log(b0008,[]), BKList; - {0, {{B, K}, _V}} -> + {0, {{B, K}, _V}} when is_binary(B); is_tuple(B) -> leveled_log:log(b0009,[B]), - get_nextbucket(leveled_codec:next_key(B), - null, - Tag, - LedgerSnapshot, - [{B, K}|BKList], - {C + 1, L}) + get_nextbucket( + leveled_codec:next_key(B), + null, + Tag, + LedgerSnapshot, + [{B, K}|BKList], + {C + 1, L} + ) end. --spec foldobjects(snap_fun(), - atom(), - list(), - fold_objects_fun()|{fold_objects_fun(), foldacc()}, - false|{true, boolean()}, false|list(integer())) -> - {async, runner_fun()}. +-spec foldobjects( + snap_fun(), + atom(), + list(), + fold_objects_fun()|{fold_objects_fun(), foldacc()}, + false|{true, boolean()}, false|list(integer())) + -> {async, runner_fun()}. foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList) -> foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, SegmentList, false, false). @@ -534,14 +540,16 @@ foldobjects(SnapFun, Tag, KeyRanges, FoldObjFun, DeferredFetch, FoldFun, JournalSnapshot, Tag, DeferredFetch), FoldFunGen = fun({StartKey, EndKey}, FoldAcc) -> - leveled_penciller:pcl_fetchkeysbysegment(LedgerSnapshot, - StartKey, - EndKey, - AccFun, - FoldAcc, - SegmentList, - LastModRange, - LimitByCount) + leveled_penciller:pcl_fetchkeysbysegment( + LedgerSnapshot, + StartKey, + EndKey, + AccFun, + FoldAcc, + SegmentList, + LastModRange, + LimitByCount + ) end, ListFoldFun = fun(KeyRange, Acc) -> @@ -567,9 +575,7 @@ accumulate_hashes(JournalCheck, InkerClone) -> fun(B, K, H, Acc) -> [{B, K, H}|Acc] end, - get_hashaccumulator(JournalCheck, - InkerClone, - AddKeyFun). + get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun). accumulate_tree(FilterFun, JournalCheck, InkerClone, HashFun) -> AddKeyFun = @@ -581,15 +587,13 @@ accumulate_tree(FilterFun, JournalCheck, InkerClone, HashFun) -> Tree end end, - get_hashaccumulator(JournalCheck, - InkerClone, - AddKeyFun). + get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun). get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun) -> AccFun = fun(LK, V, Acc) -> {B, K, H} = leveled_codec:get_keyandobjhash(LK, V), - Check = leveled_rand:uniform() < ?CHECKJOURNAL_PROB, + Check = rand:uniform() < ?CHECKJOURNAL_PROB, case JournalCheck and Check of true -> case check_presence(LK, V, InkerClone) of @@ -604,11 +608,11 @@ get_hashaccumulator(JournalCheck, InkerClone, AddKeyFun) -> end, AccFun. --spec accumulate_objects(fold_objects_fun(), - pid()|null, - leveled_codec:tag(), - false|{true, boolean()}) - -> acc_fun(). +-spec accumulate_objects + (fold_objects_fun(), pid(), leveled_head:object_tag(), false|{true, boolean()}) + -> objectacc_fun(); + (fold_objects_fun(), null, leveled_head:headonly_tag(), {true, false}) + -> objectacc_fun(). accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> AccFun = fun(LK, V, Acc) -> @@ -630,24 +634,23 @@ accumulate_objects(FoldObjectsFun, InkerClone, Tag, DeferredFetch) -> {B0, K0, _T0} -> {B0, K0} end, - JK = {leveled_codec:to_ledgerkey(B, K, Tag), SQN}, + JK = {leveled_codec:to_objectkey(B, K, Tag), SQN}, case DeferredFetch of - {true, JournalCheck} -> + {true, JournalCheck} when MD =/= null -> ProxyObj = leveled_codec:return_proxy(Tag, MD, InkerClone, JK), - case JournalCheck of - true -> + case {JournalCheck, InkerClone} of + {true, InkerClone} when is_pid(InkerClone) -> InJournal = - leveled_inker:ink_keycheck(InkerClone, - LK, - SQN), + leveled_inker:ink_keycheck( + InkerClone, LK, SQN), case InJournal of probably -> FoldObjectsFun(B, K, ProxyObj, Acc); missing -> Acc end; - false -> + {false, _} -> FoldObjectsFun(B, K, ProxyObj, Acc) end; false -> @@ -730,10 +733,10 @@ throw_test() -> fun() -> error end, - ?assertMatch({ok, ['1']}, - wrap_runner(CompletedFolder, AfterAction)), - ?assertException(throw, stop_fold, - wrap_runner(StoppedFolder, AfterAction)). + ?assertMatch({ok, ['1']}, wrap_runner(CompletedFolder, AfterAction)), + ?assertException( + throw, stop_fold, wrap_runner(StoppedFolder, AfterAction) + ). -endif. diff --git a/src/leveled_sst.erl b/src/leveled_sst.erl index 56472742..8aa21a9b 100644 --- a/src/leveled_sst.erl +++ b/src/leveled_sst.erl @@ -63,6 +63,9 @@ -include("leveled.hrl"). +% Test functions to ignore for equalizer +-eqwalizer({nowarn_function, fetch_status_test/0}). + -define(LOOK_SLOTSIZE, 128). % Maximum of 128 -define(LOOK_BLOCKSIZE, {24, 32}). % 4x + y = ?LOOK_SLOTSIZE -define(NOLOOK_SLOTSIZE, 256). @@ -83,7 +86,6 @@ -define(FLIPPER32, 4294967295). -define(DOUBLESIZE_LEVEL, 3). -define(INDEX_MODDATE, true). --define(TOMB_COUNT, true). -define(USE_SET_FOR_SPEED, 32). -define(STARTUP_TIMEOUT, 10000). -define(MIN_HASH, 32768). @@ -126,7 +128,7 @@ sst_gettombcount/1, sst_close/1]). --export([sst_newmerge/10]). +-export([sst_newmerge/9]). -export([tune_seglist/1, extract_hash/1, segment_checker/1]). @@ -170,7 +172,7 @@ range_endpoint()}. -type expandable_pointer() :: slot_pointer()|sst_pointer()|sst_closed_pointer(). --type expanded_pointer() +-type maybe_expanded_pointer() :: leveled_codec:ledger_kv()|expandable_pointer(). -type expanded_slot() :: {binary(), non_neg_integer(), range_endpoint(), range_endpoint()}. @@ -179,7 +181,12 @@ -type sst_options() :: #sst_options{}. -type binary_slot() - :: {binary(), binary(), list(integer()), leveled_codec:ledger_key()}. + :: { + binary(), + binary(), + list(leveled_codec:segment_hash()), + leveled_codec:ledger_key() + }. -type sst_summary() :: #summary{}. -type blockindex_cache() @@ -199,27 +206,38 @@ | false. -type fetch_levelzero_fun() :: fun((pos_integer(), leveled_penciller:levelzero_returnfun()) -> ok). +-type extract_hash() :: non_neg_integer()|no_lookup. + +-record(read_state, + { + handle :: file:io_device(), + blockindex_cache :: blockindex_cache()|redacted, + fetch_cache :: fetch_cache()|redacted, + level :: leveled_pmanifest:lsm_level(), + filter_fun :: summary_filter() + } +). + +-type read_state() :: #read_state{}. -record(state, - {summary, - handle :: file:fd() | undefined, - penciller :: pid() | undefined | false, - root_path, - filename, - blockindex_cache :: - blockindex_cache() | undefined | redacted, - compression_method = native :: press_method(), - index_moddate = ?INDEX_MODDATE :: boolean(), - starting_pid :: pid()|undefined, - fetch_cache = no_cache :: fetch_cache() | redacted, - new_slots :: list()|undefined, - deferred_startup_tuple :: tuple()|undefined, - level :: leveled_pmanifest:lsm_level()|undefined, - tomb_count = not_counted - :: non_neg_integer()|not_counted, - high_modified_date :: non_neg_integer()|undefined, - filter_fun :: summary_filter() | undefined, - monitor = {no_monitor, 0} :: leveled_monitor:monitor()}). + { + summary, + penciller :: pid() | undefined | false, + root_path, + filename, + read_state :: read_state() | undefined, + compression_method = native :: press_method(), + index_moddate = ?INDEX_MODDATE :: boolean(), + starting_pid :: pid()|undefined, + new_slots :: list()|undefined, + deferred_startup_tuple :: tuple()|undefined, + tomb_count = not_counted + :: non_neg_integer()|not_counted, + high_modified_date :: non_neg_integer()|undefined, + monitor = {no_monitor, 0} :: leveled_monitor:monitor() + } +). -record(build_timings, {slot_hashlist = 0 :: integer(), @@ -238,8 +256,9 @@ -spec sst_open( string(), string(), sst_options(), leveled_pmanifest:lsm_level()) - -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, + -> + {ok, pid(), + {leveled_codec:object_key(), leveled_codec:object_key()}, binary()}. %% @doc %% Open an SST file at a given path and filename. The first and last keys @@ -261,7 +280,8 @@ sst_open(RootPath, Filename, OptsSST, Level) -> list(leveled_codec:ledger_kv()), integer(), sst_options()) -> {ok, pid(), - {leveled_codec:ledger_key(), leveled_codec:ledger_key()}, + {leveled_codec:object_key(), + leveled_codec:object_key()}, binary()}. %% @doc %% Start a new SST file at the assigned level passing in a list of Key, Value @@ -299,8 +319,8 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN, OptsSST, IndexModDate) -> -> empty|{ok, pid(), {{list(leveled_codec:ledger_kv()), list(leveled_codec:ledger_kv())}, - leveled_codec:ledger_key(), - leveled_codec:ledger_key()}, + leveled_codec:object_key(), + leveled_codec:object_key()}, binary()}. %% @doc %% Start a new SST file at the assigned level passing in a two lists of @@ -318,11 +338,11 @@ sst_newmerge(RootPath, Filename, MaxSQN, OptsSST) when Level > 0 -> sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, - MaxSQN, OptsSST, ?INDEX_MODDATE, ?TOMB_COUNT). + MaxSQN, OptsSST, ?INDEX_MODDATE). sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, - MaxSQN, OptsSST, IndexModDate, TombCount) -> + MaxSQN, OptsSST, IndexModDate) -> OptsSST0 = update_options(OptsSST, Level), {Rem1, Rem2, SlotList, FK, CountOfTombs} = merge_lists( @@ -330,8 +350,8 @@ sst_newmerge(RootPath, Filename, KVL2, {IsBasement, Level}, OptsSST0, - IndexModDate, - TombCount), + IndexModDate + ), case SlotList of [] -> empty; @@ -348,7 +368,9 @@ sst_newmerge(RootPath, Filename, MaxSQN, OptsSST0, IndexModDate, - CountOfTombs, self()}, + CountOfTombs, + self() + }, infinity), {ok, Pid, {{Rem1, Rem2}, SK, EK}, Bloom} end. @@ -387,8 +409,9 @@ sst_newlevelzero( {ok, Pid, noreply}. --spec sst_get(pid(), leveled_codec:ledger_key()) - -> leveled_codec:ledger_kv()|not_present. +-spec sst_get( + pid(), leveled_codec:object_key()) -> + leveled_codec:ledger_kv()|not_present. %% @doc %% Return a Key, Value pair matching a Key or not_present if the Key is not in %% the store. The segment_hash function is used to accelerate the seeking of @@ -396,17 +419,18 @@ sst_newlevelzero( sst_get(Pid, LedgerKey) -> sst_get(Pid, LedgerKey, leveled_codec:segment_hash(LedgerKey)). --spec sst_get(pid(), leveled_codec:ledger_key(), leveled_codec:segment_hash()) - -> leveled_codec:ledger_kv()|not_present. +-spec sst_get( + pid(), leveled_codec:object_key(), leveled_codec:segment_hash()) + -> leveled_codec:ledger_kv()|not_present. %% @doc %% Return a Key, Value pair matching a Key or not_present if the Key is not in %% the store (with the magic hash precalculated). sst_get(Pid, LedgerKey, Hash) -> gen_statem:call(Pid, {get_kv, LedgerKey, Hash, undefined}, infinity). --spec sst_getsqn(pid(), - leveled_codec:ledger_key(), - leveled_codec:segment_hash()) -> leveled_codec:sqn()|not_present. +-spec sst_getsqn( + pid(), leveled_codec:object_key(), leveled_codec:segment_hash()) + -> leveled_codec:sqn()|not_present. %% @doc %% Return a SQN for the key or not_present if the key is not in %% the store (with the magic hash precalculated). @@ -424,7 +448,7 @@ sst_getmaxsequencenumber(Pid) -> list(expandable_pointer()), pos_integer(), segment_check_fun(), - non_neg_integer()) -> list(expanded_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Expand out a list of pointer to return a list of Keys and Values with a %% tail of pointers (once the ScanWidth has been satisfied). @@ -469,7 +493,7 @@ sst_deleteconfirmed(Pid) -> gen_statem:cast(Pid, close). -spec sst_checkready(pid()) -> - {ok, string(), leveled_codec:ledger_key(), leveled_codec:ledger_key()}. + {ok, string(), leveled_codec:object_key(), leveled_codec:object_key()}. %% @doc %% If a file has been set to be built, check that it has been built. Returns %% the filename and the {startKey, EndKey} for the manifest. @@ -507,14 +531,17 @@ starting({call, From}, leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, {UpdState, Bloom} = - read_file(Filename, - State#state{root_path=RootPath}, - OptsSST#sst_options.pagecache_level >= Level), + read_file( + Filename, + State#state{root_path=RootPath}, + OptsSST#sst_options.pagecache_level >= Level, + undefined, + Level + ), Summary = UpdState#state.summary, {next_state, reader, - UpdState#state{ - level = Level, fetch_cache = new_cache(Level), monitor = Monitor}, + UpdState#state{monitor = Monitor}, [{reply, From, {ok, {Summary#summary.first_key, Summary#summary.last_key}, @@ -532,7 +559,7 @@ starting({call, From}, PressMethod = OptsSST#sst_options.press_method, {Length, SlotIndex, BlockEntries, SlotsBin, Bloom} = build_all_slots(SlotList), - {_, BlockIndex, HighModDate} = + {_, BlockIndexCache, HighModDate} = update_blockindex_cache( BlockEntries, new_blockindex_cache(Length), @@ -549,7 +576,10 @@ starting({call, From}, read_file( ActualFilename, State#state{root_path=RootPath}, - OptsSST#sst_options.pagecache_level >= Level), + OptsSST#sst_options.pagecache_level >= Level, + BlockIndexCache, + Level + ), Summary = UpdState#state.summary, leveled_log:log_timer( sst08, [ActualFilename, Level, Summary#summary.max_sqn], SW), @@ -557,11 +587,8 @@ starting({call, From}, {next_state, reader, UpdState#state{ - blockindex_cache = BlockIndex, high_modified_date = HighModDate, starting_pid = StartingPID, - level = Level, - fetch_cache = new_cache(Level), monitor = Monitor}, [{reply, From, @@ -575,9 +602,7 @@ starting({call, From}, {sst_newlevelzero, RootPath, Filename, IdxModDate}, {next_state, starting, State#state{ - deferred_startup_tuple = DeferredStartupTuple, - level = 0, - fetch_cache = new_cache(0)}, + deferred_startup_tuple = DeferredStartupTuple}, [{reply, From, ok}]}; starting({call, From}, close, State) -> %% No file should have been created, so nothing to close. @@ -592,6 +617,7 @@ starting(cast, complete_l0startup, State) -> State#state.deferred_startup_tuple, SW0 = os:timestamp(), FetchedSlots = State#state.new_slots, + FetchedSlots =/= undefined orelse error(invalid_type), leveled_log:save(OptsSST#sst_options.log_options), Monitor = OptsSST#sst_options.monitor, PressMethod = OptsSST#sst_options.press_method, @@ -607,7 +633,7 @@ starting(cast, complete_l0startup, State) -> SW2 = os:timestamp(), {SlotCount, SlotIndex, BlockEntries, SlotsBin,Bloom} = build_all_slots(SlotList), - {_, BlockIndex, HighModDate} = + {_, BlockIndexCache, HighModDate} = update_blockindex_cache( BlockEntries, new_blockindex_cache(SlotCount), @@ -632,7 +658,10 @@ starting(cast, complete_l0startup, State) -> root_path=RootPath, new_slots=undefined, % Important to empty this from state deferred_startup_tuple=undefined}, - true), + true, + BlockIndexCache, + 0 + ), Summary = UpdState#state.summary, Time4 = timer:now_diff(os:timestamp(), SW4), @@ -655,16 +684,15 @@ starting(cast, complete_l0startup, State) -> {next_state, reader, UpdState#state{ - blockindex_cache = BlockIndex, high_modified_date = HighModDate, monitor = Monitor}}; starting(cast, {sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> FetchedSlots = - case FetchedSlot of - none -> - []; + case {FetchedSlot, State#state.new_slots} of + {FS, PreviousSlots} when FS =/= none, is_list(PreviousSlots) -> + [FetchedSlot|PreviousSlots]; _ -> - [FetchedSlot|State#state.new_slots] + [] end, case length(FetchedSlots) == SlotCount of true -> @@ -685,7 +713,9 @@ starting(cast, {sst_returnslot, FetchedSlot, FetchFun, SlotCount}, State) -> State#state{new_slots = FetchedSlots}} end. -reader({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> +reader({call, From}, + {get_kv, LedgerKey, Hash, Filter}, + State = #state{read_state = RS}) when ?IS_DEF(RS)-> % Get a KV value and potentially take sample timings Monitor = case Filter of @@ -694,65 +724,69 @@ reader({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> _ -> {no_monitor, 0} end, - {KeyValue, BIC, HMD, FC} = + FetchResult = fetch( - LedgerKey, Hash, + LedgerKey, + Hash, State#state.summary, State#state.compression_method, State#state.high_modified_date, State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, + RS#read_state.filter_fun, + RS#read_state.blockindex_cache, + RS#read_state.fetch_cache, + RS#read_state.handle, + RS#read_state.level, Monitor), - Result = + FilterFun = case Filter of - undefined -> - KeyValue; - F -> - F(KeyValue) + undefined -> fun(KV) -> KV end; + F when is_function(F) -> F end, - case {BIC, HMD, FC} of - {no_update, no_update, no_update} -> - {keep_state_and_data, [{reply, From, Result}]}; - {no_update, no_update, FC} -> + case FetchResult of + {KV, no_update, no_update, no_update} -> + {keep_state_and_data, [{reply, From, FilterFun(KV)}]}; + {KV, no_update, no_update, FC} -> + RS0 = RS#read_state{fetch_cache = FC}, {keep_state, - State#state{fetch_cache = FC}, - [{reply, From, Result}]}; - {BIC, undefined, no_update} -> + State#state{read_state = RS0}, + [{reply, From, FilterFun(KV)}]}; + {KV, BIC, undefined, no_update} when BIC =/= no_update -> + RS0 = RS#read_state{blockindex_cache = BIC}, {keep_state, - State#state{blockindex_cache = BIC}, - [{reply, From, Result}]}; - {BIC, HMD, no_update} -> + State#state{read_state = RS0}, + [{reply, From, FilterFun(KV)}]}; + {KV, BIC, HMD, no_update} when BIC =/= no_update, HMD =/= no_update -> + RS0 = RS#read_state{blockindex_cache = BIC}, {keep_state, - State#state{blockindex_cache = BIC, high_modified_date = HMD}, - [hibernate, {reply, From, Result}]} + State#state{read_state = RS0, high_modified_date = HMD}, + [hibernate, {reply, From, FilterFun(KV)}]} end; reader({call, From}, {fetch_range, StartKey, EndKey, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> SlotsToPoint = fetch_range( StartKey, EndKey, State#state.summary, - State#state.filter_fun, + RS#read_state.filter_fun, check_modified( State#state.high_modified_date, LowLastMod, State#state.index_moddate) ), {keep_state_and_data, [{reply, From, SlotsToPoint}]}; -reader({call, From}, {get_slots, SlotList, SegChecker, LowLastMod}, State) -> +reader({call, From}, + {get_slots, SlotList, SegChecker, LowLastMod}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, {NeedBlockIdx, SlotBins} = read_slots( - State#state.handle, + RS#read_state.handle, SlotList, - {SegChecker, LowLastMod, State#state.blockindex_cache}, + {SegChecker, LowLastMod, RS#read_state.blockindex_cache}, State#state.compression_method, State#state.index_moddate), {keep_state_and_data, @@ -780,15 +814,22 @@ reader({call, From}, background_complete, State) -> reader({call, From}, get_tomb_count, State) -> {keep_state_and_data, [{reply, From, State#state.tomb_count}]}; -reader({call, From}, close, State) -> - ok = file:close(State#state.handle), +reader({call, From}, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> + ok = file:close(RS#read_state.handle), {stop_and_reply, normal, [{reply, From, ok}], State}; -reader(cast, {switch_levels, NewLevel}, State) -> +reader(cast, + {switch_levels, NewLevel}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {keep_state, State#state{ - level = NewLevel, - fetch_cache = new_cache(NewLevel) + read_state = + RS#read_state{ + fetch_cache = new_cache(NewLevel), + level = NewLevel + } }, [hibernate]}; reader(info, {update_blockindex_cache, BIC}, State) -> @@ -799,7 +840,10 @@ reader(info, bic_complete, State) -> % fragmentation leveled_log:log(sst14, [State#state.filename]), {keep_state_and_data, [hibernate]}; -reader(info, start_complete, State) -> +reader( + info, + start_complete, + #state{starting_pid = StartingPid}) when ?IS_DEF(StartingPid) -> % The SST file will be started by a clerk, but the clerk may be shut down % prior to the manifest being updated about the existence of this SST file. % If there is no activity after startup, check the clerk is still alive and @@ -807,7 +851,7 @@ reader(info, start_complete, State) -> % If the clerk has crashed, the penciller will restart at the latest % manifest, and so this file sill be restarted if and only if it is still % part of the store - case is_process_alive(State#state.starting_pid) of + case is_process_alive(StartingPid) of true -> {keep_state_and_data, []}; false -> @@ -815,7 +859,9 @@ reader(info, start_complete, State) -> end. -delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> +delete_pending({call, From}, + {get_kv, LedgerKey, Hash, Filter}, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {KeyValue, _BIC, _HMD, _FC} = fetch( LedgerKey, Hash, @@ -823,11 +869,11 @@ delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> State#state.compression_method, State#state.high_modified_date, State#state.index_moddate, - State#state.filter_fun, - State#state.blockindex_cache, - State#state.fetch_cache, - State#state.handle, - State#state.level, + RS#read_state.filter_fun, + RS#read_state.blockindex_cache, + RS#read_state.fetch_cache, + RS#read_state.handle, + RS#read_state.level, {no_monitor, 0}), Result = case Filter of @@ -840,13 +886,13 @@ delete_pending({call, From}, {get_kv, LedgerKey, Hash, Filter}, State) -> delete_pending( {call, From}, {fetch_range, StartKey, EndKey, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> SlotsToPoint = fetch_range( StartKey, EndKey, State#state.summary, - State#state.filter_fun, + RS#read_state.filter_fun, check_modified( State#state.high_modified_date, LowLastMod, @@ -856,29 +902,35 @@ delete_pending( delete_pending( {call, From}, {get_slots, SlotList, SegChecker, LowLastMod}, - State) -> + State = #state{read_state = RS}) when ?IS_DEF(RS) -> PressMethod = State#state.compression_method, IdxModDate = State#state.index_moddate, {_NeedBlockIdx, SlotBins} = read_slots( - State#state.handle, + RS#read_state.handle, SlotList, - {SegChecker, LowLastMod, State#state.blockindex_cache}, + {SegChecker, LowLastMod, RS#read_state.blockindex_cache}, PressMethod, IdxModDate), {keep_state_and_data, [{reply, From, {false, SlotBins, PressMethod, IdxModDate}}, ?DELETE_TIMEOUT]}; -delete_pending({call, From}, close, State) -> +delete_pending( + {call, From}, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), + ok = file:close(RS#read_state.handle), ok = file:delete( filename:join(State#state.root_path, State#state.filename)), {stop_and_reply, normal, [{reply, From, ok}], State}; -delete_pending(cast, close, State) -> +delete_pending( + cast, + close, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> leveled_log:log(sst07, [State#state.filename]), - ok = file:close(State#state.handle), + ok = file:close(RS#read_state.handle), ok = file:delete( filename:join(State#state.root_path, State#state.filename)), @@ -892,27 +944,32 @@ delete_pending(timeout, _, State) -> case State#state.penciller of false -> ok = leveled_sst:sst_deleteconfirmed(self()); - PCL -> + PCL when is_pid(PCL) -> FN = State#state.filename, ok = leveled_penciller:pcl_confirmdelete(PCL, FN, self()) - end, + end, % If the next thing is another timeout - may be long-running snapshot, so % back-off - {keep_state_and_data, [leveled_rand:uniform(10) * ?DELETE_TIMEOUT]}. + {keep_state_and_data, [rand:uniform(10) * ?DELETE_TIMEOUT]}. -handle_update_blockindex_cache(BIC, State) -> +handle_update_blockindex_cache( + BIC, + State = #state{read_state = RS}) when ?IS_DEF(RS) -> {NeedBlockIdx, BlockIndexCache, HighModDate} = update_blockindex_cache( BIC, - State#state.blockindex_cache, + RS#read_state.blockindex_cache, State#state.high_modified_date, State#state.index_moddate), case NeedBlockIdx of true -> {keep_state, State#state{ - blockindex_cache = BlockIndexCache, - high_modified_date = HighModDate}}; + read_state = + RS#read_state{blockindex_cache = BlockIndexCache}, + high_modified_date = HighModDate + } + }; false -> keep_state_and_data end. @@ -927,14 +984,17 @@ code_change(_OldVsn, StateName, State, _Extra) -> format_status(Status) -> - case maps:get(reason, Status, normal) of - terminate -> - State = maps:get(state, Status), + case {maps:get(reason, Status, normal), maps:get(state, Status)} of + {terminate, State = #state{read_state = RS}} when ?IS_DEF(RS) -> maps:update( state, State#state{ - blockindex_cache = redacted, - fetch_cache = redacted}, + read_state = + RS#read_state{ + blockindex_cache = redacted, + fetch_cache = redacted + } + }, Status ); _ -> @@ -949,7 +1009,7 @@ format_status(Status) -> -spec expand_list_by_pointer( expandable_pointer(), list(expandable_pointer()), - pos_integer()) -> list(expanded_pointer()). + pos_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Expand a list of pointers, maybe ending up with a list of keys and values %% with a tail of pointers @@ -966,7 +1026,7 @@ expand_list_by_pointer(Pointer, Tail, Width) -> list(expandable_pointer()), pos_integer(), segment_check_fun(), - non_neg_integer()) -> list(expanded_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% With filters (as described in expand_list_by_pointer/3 expand_list_by_pointer( @@ -975,17 +1035,7 @@ expand_list_by_pointer( {PotentialPointers, Remainder} = lists:split(min(Width - 1, length(Tail)), Tail), {LocalPointers, OtherPointers} = - lists:partition( - fun(Pointer) -> - case Pointer of - {pointer, SSTPid, _S, _SK, _EK} -> - true; - _ -> - false - end - end, - PotentialPointers - ), + split_localpointers(SSTPid, PotentialPointers), sst_getfilteredslots( SSTPid, [{pointer, SSTPid, Slot, StartKey, EndKey}|LocalPointers], @@ -994,18 +1044,30 @@ expand_list_by_pointer( OtherPointers ++ Remainder ); expand_list_by_pointer( - {next, ManEntry, StartKey, EndKey}, - Tail, _Width, _SegChecker, LowLastMod) -> + {next, ME, StartKey, EndKey}, Tail, _Width, _SegChecker, LowLastMod + ) -> % The first pointer is a pointer to a file - expand_list_by_pointer will % in this case convert this into list of pointers within that SST file % i.e. of the form {pointer, SSTPid, Slot, StartKey, EndKey} % This can then be further expanded by calling again to % expand_list_by_pointer - SSTPid = ManEntry#manifest_entry.owner, + SSTPid = leveled_pmanifest:entry_owner(ME), leveled_log:log(sst10, [SSTPid, is_process_alive(SSTPid)]), ExpPointer = sst_getfilteredrange(SSTPid, StartKey, EndKey, LowLastMod), ExpPointer ++ Tail. +-spec split_localpointers( + pid(), list(expandable_pointer())) -> + {list(slot_pointer()), list(expandable_pointer())}. +split_localpointers(LocalPid, PotentialPointers) -> + lists:partition( + fun({pointer, PID, _S, _SK, _EK}) when PID == LocalPid -> + true; + (_) -> + false + end, + PotentialPointers + ). -spec sst_getfilteredrange( pid(), @@ -1112,7 +1174,7 @@ segment_checker(HashList) when is_list(HashList) -> Max = lists:max(HashList), case length(HashList) > ?USE_SET_FOR_SPEED of true -> - HashSet = sets:from_list(HashList), + HashSet = sets:from_list(HashList, [{version, 2}]), {Min, Max, fun(H) -> sets:is_element(H, HashSet) end}; false -> {Min, Max, fun(H) -> lists:member(H, HashList) end} @@ -1128,7 +1190,7 @@ sqn_only(KV) -> leveled_codec:strip_to_seqonly(KV). -spec extract_hash( - leveled_codec:segment_hash()) -> non_neg_integer()|no_lookup. + leveled_codec:segment_hash()) -> extract_hash(). extract_hash({SegHash, _ExtraHash}) when is_integer(SegHash) -> tune_hash(SegHash); extract_hash(NotHash) -> @@ -1141,7 +1203,7 @@ new_cache(Level) -> no_cache -> no_cache; CacheSize -> - array:new([{size, CacheSize}]) + array:new([{size, CacheSize}, {default, none}]) end. -spec cache_hash(leveled_codec:segment_hash(), non_neg_integer()) -> @@ -1171,23 +1233,59 @@ cache_size(6) -> cache_size(_LowerLevel) -> no_cache. --spec fetch_from_cache( +-spec get_from_fetchcache( cache_hash(), - fetch_cache()) -> undefined|leveled_codec:ledger_kv(). -fetch_from_cache(_CacheHash, no_cache) -> - undefined; -fetch_from_cache(CacheHash, Cache) -> + fetch_cache()) -> none|leveled_codec:ledger_kv(). +get_from_fetchcache(_CacheHash, no_cache) -> + none; +get_from_fetchcache(CacheHash, Cache) when is_integer(CacheHash) -> + % When defining array can use array:array/1 but this will lead to type + % issues when using array:new + % eqwalizer:ignore array:get(CacheHash, Cache). -spec add_to_cache( - non_neg_integer(), + non_neg_integer()|no_cache, leveled_codec:ledger_kv(), fetch_cache()) -> fetch_cache(). -add_to_cache(_CacheHash, _KV, no_cache) -> - no_cache; -add_to_cache(CacheHash, KV, FetchCache) -> - array:set(CacheHash, KV, FetchCache). +add_to_cache( + CacheHash, KV, FetchCache) + when is_integer(CacheHash), FetchCache =/= no_cache -> + array:set(CacheHash, KV, FetchCache); +add_to_cache(_CacheHash, _KV, _FetchCache) -> + no_cache. + +-spec get_from_blockcache(pos_integer(), blockindex_cache()) -> binary()|none. +get_from_blockcache(SlotID, BIC) when is_integer(SlotID) -> + case array:get(SlotID - 1, element(2, BIC)) of + CBI when is_binary(CBI) -> + CBI; + none -> + none + end. + +-spec add_to_blockcache( + pos_integer(), blockindex_cache(), binary(), non_neg_integer()|false) -> + blockindex_cache(). +add_to_blockcache(SlotID, {Cnt, Cache, HighLMD}, Block, LMD) -> + { + Cnt + 1, + array:set(SlotID - 1, binary:copy(Block), Cache), + case LMD of + LMD when is_integer(LMD), LMD > HighLMD -> + LMD; + _ -> + HighLMD + end + }. +-spec size_of_blockcache(blockindex_cache()) -> non_neg_integer(). +size_of_blockcache(BIC) -> + array:size(element(2, BIC)). + +-spec new_blockindex_cache(pos_integer()) -> blockindex_cache(). +new_blockindex_cache(Size) -> + {0, array:new([{size, Size}, {default, none}]), 0}. -spec tune_hash(non_neg_integer()) -> ?MIN_HASH..?MAX_HASH. %% @doc @@ -1220,44 +1318,40 @@ update_options(OptsSST, Level) -> maxslots_level(Level, OptsSST#sst_options.max_sstslots), OptsSST#sst_options{press_method = PressMethod0, max_sstslots = MaxSlots0}. --spec new_blockindex_cache(pos_integer()) -> blockindex_cache(). -new_blockindex_cache(Size) -> - {0, array:new([{size, Size}, {default, none}]), 0}. - -spec updatebic_foldfun(boolean()) -> - fun(({integer(), binary()}, blockindex_cache()) -> blockindex_cache()). + fun(({integer(), binary()|none}, blockindex_cache()) -> blockindex_cache()). updatebic_foldfun(HMDRequired) -> - fun(CacheEntry, {AccCount, Cache, AccHMD}) -> + fun(CacheEntry, BIC) -> case CacheEntry of {ID, Header} when is_binary(Header) -> - case array:get(ID - 1, Cache) of + case get_from_blockcache(ID, BIC) of none -> - H0 = binary:copy(Header), - AccHMD0 = + LMD = case HMDRequired of true -> - max(AccHMD, - element(2, extract_header(H0, true))); + {_, ExtractLMD, _} = + extract_header(Header, true), + ExtractLMD; false -> - AccHMD + false end, - {AccCount + 1, array:set(ID - 1, H0, Cache), AccHMD0}; + add_to_blockcache(ID, BIC, Header, LMD); _ -> - {AccCount, Cache, AccHMD} + BIC end; _ -> - {AccCount, Cache, AccHMD} + BIC end end. -spec update_blockindex_cache( - list({integer(), binary()}), + list({integer(), binary()|none}), blockindex_cache(), non_neg_integer()|undefined, boolean()) -> {boolean(), blockindex_cache(), non_neg_integer()|undefined}. update_blockindex_cache(Entries, BIC, HighModDate, IdxModDate) -> - case {element(1, BIC), array:size(element(2, BIC))} of + case {element(1, BIC), size_of_blockcache(BIC)} of {N, N} -> {false, BIC, HighModDate}; {N, S} when N < S -> @@ -1318,7 +1412,7 @@ fetch(LedgerKey, Hash, Slot = lookup_slot(LedgerKey, Summary#summary.index, FilterFun), SlotID = Slot#slot_index_value.slot_id, - CachedBlockIdx = array:get(SlotID - 1, element(2, BIC)), + CachedBlockIdx = get_from_blockcache(SlotID, BIC), case extract_header(CachedBlockIdx, IndexModDate) of none -> @@ -1340,14 +1434,17 @@ fetch(LedgerKey, Hash, {Result, BIC0, HMD0, no_update}; {BlockLengths, _LMD, PosBin} -> PosList = - find_pos(PosBin, segment_checker(extract_hash(Hash))), + case extract_hash(Hash) of + HashExtract when is_integer(HashExtract) -> + find_pos(PosBin, segment_checker(HashExtract)) + end, case PosList of [] -> maybelog_fetch_timing(Monitor, Level, not_found, SW0), {not_present, no_update, no_update, no_update}; _ -> CacheHash = cache_hash(Hash, Level), - case fetch_from_cache(CacheHash, FetchCache) of + case get_from_fetchcache(CacheHash, FetchCache) of {LedgerKey, V} -> maybelog_fetch_timing( Monitor, Level, fetch_cache, SW0), @@ -1355,29 +1452,32 @@ fetch(LedgerKey, Hash, _ -> StartPos = Slot#slot_index_value.start_position, Result = - check_blocks( + check_blocks_matchkey( PosList, {Handle, StartPos}, BlockLengths, byte_size(PosBin), LedgerKey, PressMethod, - IndexModDate, - not_present), + IndexModDate + ), case Result of not_present -> maybelog_fetch_timing( Monitor, Level, not_found, SW0), {not_present, no_update, no_update, no_update}; - _ -> + {LK, LV} -> FetchCache0 = add_to_cache( - CacheHash, Result, FetchCache), + CacheHash, {LK, LV}, FetchCache), maybelog_fetch_timing( Monitor, Level, slot_cachedblock, SW0), - {Result, - no_update, no_update, FetchCache0} + {{LK, LV}, + no_update, + no_update, + FetchCache0 + } end end end @@ -1461,7 +1561,7 @@ write_file(RootPath, Filename, SummaryBin, SlotsBin, false), FinalName. -read_file(Filename, State, LoadPageCache) -> +read_file(Filename, State, LoadPageCache, BIC, Level) -> {Handle, FileVersion, SummaryBin} = open_reader( filename:join(State#state.root_path, Filename), @@ -1469,19 +1569,33 @@ read_file(Filename, State, LoadPageCache) -> UpdState0 = imp_fileversion(FileVersion, State), {Summary, Bloom, SlotList, TombCount} = read_table_summary(SummaryBin, UpdState0#state.tomb_count), - BlockIndexCache = new_blockindex_cache(Summary#summary.size), - UpdState1 = UpdState0#state{blockindex_cache = BlockIndexCache}, {SlotIndex, FilterFun} = from_list( SlotList, Summary#summary.first_key, Summary#summary.last_key), UpdSummary = Summary#summary{index = SlotIndex}, leveled_log:log( sst03, [Filename, Summary#summary.size, Summary#summary.max_sqn]), - {UpdState1#state{summary = UpdSummary, - handle = Handle, - filename = Filename, - tomb_count = TombCount, - filter_fun = FilterFun}, + ReadState = + #read_state{ + handle = Handle, + blockindex_cache = + case BIC of + undefined -> + new_blockindex_cache(Summary#summary.size); + _ -> + BIC + end, + fetch_cache = new_cache(Level), + level = Level, + filter_fun = FilterFun + }, + + {UpdState0#state{ + summary = UpdSummary, + filename = Filename, + tomb_count = TombCount, + read_state = ReadState + }, Bloom}. gen_fileversion(PressMethod, IdxModDate, CountOfTombs) -> @@ -1678,7 +1792,7 @@ serialise_block(Term, none) -> CRC32 = hmac(Bin), <>. --spec deserialise_block(binary(), press_method()) -> any(). +-spec deserialise_block(binary(), press_method()) -> list(leveled_codec:ledger_kv()). %% @doc %% Convert binary to term %% Function split out to make it easier to experiment with different @@ -1688,21 +1802,27 @@ serialise_block(Term, none) -> deserialise_block(Bin, PressMethod) when byte_size(Bin) > 4 -> BinS = byte_size(Bin) - 4, <> = Bin, - case hmac(TermBin) of - CRC32 -> - deserialise_checkedblock(TermBin, PressMethod); - _ -> + try + CRC32 = hmac(TermBin), + deserialise_checkedblock(TermBin, PressMethod) + catch + _Exception:_Reason -> [] end; deserialise_block(_Bin, _PM) -> []. -deserialise_checkedblock(Bin, lz4) -> - {ok, Bin0} = lz4:unpack(Bin), - binary_to_term(Bin0); -deserialise_checkedblock(Bin, zstd) -> - binary_to_term(zstd:decompress(Bin)); -deserialise_checkedblock(Bin, _Other) -> +deserialise_checkedblock(Bin, lz4) when is_binary(Bin) -> + case lz4:unpack(Bin) of + {ok, Bin0} when is_binary(Bin0) -> + binary_to_term(Bin0) + end; +deserialise_checkedblock(Bin, zstd) when is_binary(Bin) -> + case zstd:decompress(Bin) of + Bin0 when is_binary(Bin0) -> + binary_to_term(Bin0) + end; +deserialise_checkedblock(Bin, _Other) when is_binary(Bin) -> % native or none can be treated the same binary_to_term(Bin). @@ -1766,7 +1886,8 @@ null_filter(Key) -> Key. key_filter({_Tag, _Bucket, Key, null}) -> Key. -spec term_filter(leveled_codec:ledger_key()) -> leveled_codec:slimmed_key(). -term_filter({_Tag, _Bucket, {_Field, Term}, Key}) -> {Term, Key}. +term_filter({_Tag, _Bucket, {_Field, Term}, Key}) -> + {Term, Key}. -spec key_prefix_filter( pos_integer(), binary()) -> @@ -1868,12 +1989,13 @@ lookup_slots(StartKey, EndKey, Tree, FilterFun) -> {binary(), non_neg_integer(), list(leveled_codec:segment_hash()), - leveled_codec:last_moddate()}. + non_neg_integer()}. %% @doc %% Fold function use to accumulate the position information needed to %% populate the summary of the slot -accumulate_positions([], Acc) -> - Acc; +accumulate_positions( + [], {PosBin, NoHashCount, HashAcc, LMDAcc}) when is_integer(LMDAcc) -> + {PosBin, NoHashCount, HashAcc, LMDAcc}; accumulate_positions([{K, V}|T], {PosBin, NoHashCount, HashAcc, LMDAcc}) -> {_SQN, H1, LMD} = leveled_codec:strip_to_indexdetails({K, V}), LMDAcc0 = take_max_lastmoddate(LMD, LMDAcc), @@ -2053,52 +2175,95 @@ generate_binary_slot( {{Header, SlotBin, HashL, LastKey}, BuildTimings3}. --spec check_blocks(list(integer()), - binary()|{file:io_device(), integer()}, - binary(), - integer(), - leveled_codec:ledger_key()|false, - %% if false the acc is a list, and if true - %% Acc will be initially not_present, and may - %% result in a {K, V} tuple - press_method(), - boolean(), - list()|not_present) -> - list(leveled_codec:ledger_kv())| - not_present|leveled_codec:ledger_kv(). +-spec check_blocks_allkeys( + list(integer()), + binary()|{file:io_device(), integer()}, + binary(), + integer(), + press_method(), + boolean(), + list()) -> + list(leveled_codec:ledger_kv()). %% @doc %% Acc should start as not_present if LedgerKey is a key, and a list if %% LedgerKey is false -check_blocks([], _BlockPointer, _BlockLengths, _PosBinLength, - _LedgerKeyToCheck, _PressMethod, _IdxModDate, not_present) -> - not_present; -check_blocks([], _BlockPointer, _BlockLengths, _PosBinLength, - _LedgerKeyToCheck, _PressMethod, _IdxModDate, Acc) -> +check_blocks_allkeys([], _BP, _BLs, _PBL, _PM, _IMD, Acc) -> lists:reverse(Acc); -check_blocks([Pos|Rest], BlockPointer, BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, Acc) -> +check_blocks_allkeys( + [Pos|Rest], + BlockPointer, + BlockLengths, + PosBinLength, + PressMethod, + IdxModDate, + Acc) -> {BlockNumber, BlockPos} = revert_position(Pos), BlockBin = - read_block(BlockPointer, - BlockLengths, - PosBinLength, - BlockNumber, - additional_offset(IdxModDate)), - Result = spawn_check_block(BlockPos, BlockBin, PressMethod), - case {Result, LedgerKeyToCheck} of + read_block( + BlockPointer, + BlockLengths, + PosBinLength, + BlockNumber, + additional_offset(IdxModDate) + ), + case spawn_check_block(BlockPos, BlockBin, PressMethod) of + {K, V} -> + check_blocks_allkeys( + Rest, + BlockPointer, + BlockLengths, + PosBinLength, + PressMethod, + IdxModDate, + [{K, V}|Acc] + ) + end. + +-spec check_blocks_matchkey( + list(integer()), + binary()|{file:io_device(), integer()}, + binary(), + integer(), + leveled_codec:ledger_key(), + press_method(), + boolean()) -> + not_present|leveled_codec:ledger_kv(). +%% @doc +%% Acc should start as not_present if LedgerKey is a key, and a list if +%% LedgerKey is false +check_blocks_matchkey([], _BP, _BLs, _PBL, _LKTC, _PM, _IMD) -> + not_present; +check_blocks_matchkey( + [Pos|Rest], + BlockPointer, + BlockLengths, + PosBinLength, + LedgerKeyToCheck, + PressMethod, + IdxModDate) -> + {BlockNumber, BlockPos} = revert_position(Pos), + BlockBin = + read_block(BlockPointer, + BlockLengths, + PosBinLength, + BlockNumber, + additional_offset(IdxModDate) + ), + CheckResult = spawn_check_block(BlockPos, BlockBin, PressMethod), + case {CheckResult, LedgerKeyToCheck} of {{K, V}, K} -> {K, V}; - {{K, V}, false} -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, - [{K, V}|Acc]); _ -> - check_blocks(Rest, BlockPointer, - BlockLengths, PosBinLength, - LedgerKeyToCheck, PressMethod, IdxModDate, - Acc) - end. + check_blocks_matchkey( + Rest, + BlockPointer, + BlockLengths, + PosBinLength, + LedgerKeyToCheck, + PressMethod, + IdxModDate + ) +end. -spec spawn_check_block(non_neg_integer(), binary(), press_method()) -> not_present|leveled_codec:ledger_kv(). @@ -2204,7 +2369,7 @@ read_slots(Handle, SlotList, {SegChecker, LowLastMod, BlockIndexCache}, BinMapFun = fun(Pointer, {NeededBlockIdx, Acc}) -> {SP, _L, ID, SK, EK} = pointer_mapfun(Pointer), - CachedHeader = array:get(ID - 1, element(2, BlockIndexCache)), + CachedHeader = get_from_blockcache(ID, BlockIndexCache), case extract_header(CachedHeader, IdxModDate) of none -> % If there is an attempt to use the seg list query and the @@ -2271,9 +2436,15 @@ checkblocks_segandrange( PressMethod, IdxModDate, SegChecker, {StartKey, EndKey}) -> PositionList = find_pos(BlockIdx, SegChecker), KVL = - check_blocks( - PositionList, SlotOrHandle, BlockLengths, byte_size(BlockIdx), - false, PressMethod, IdxModDate, []), + check_blocks_allkeys( + PositionList, + SlotOrHandle, + BlockLengths, + byte_size(BlockIdx), + PressMethod, + IdxModDate, + [] + ), in_range(KVL, StartKey, EndKey). @@ -2371,18 +2542,26 @@ extract_header(Header, false) -> <> = Header, {BlockLengths, 0, PosBinIndex}. +-spec binaryslot_get( + binary(), + leveled_codec:ledger_key(), + leveled_codec:segment_hash(), + press_method(), + boolean()) -> {not_present|leveled_codec:ledger_kv(), binary()|none}. binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> case crc_check_slot(FullBin) of {Header, Blocks} -> {BlockLengths, _LMD, PosBinIndex} = extract_header(Header, IdxModDate), PosList = - find_pos(PosBinIndex, segment_checker(extract_hash(Hash))), + case extract_hash(Hash) of + HashExtract when is_integer(HashExtract) -> + find_pos(PosBinIndex, segment_checker(HashExtract)) + end, {fetch_value(PosList, BlockLengths, Blocks, Key, PressMethod), Header}; crc_wonky -> - {not_present, - none} + {not_present, none} end. -spec binaryslot_tolist( @@ -2390,7 +2569,7 @@ binaryslot_get(FullBin, Key, Hash, PressMethod, IdxModDate) -> press_method(), boolean(), list(leveled_codec:ledger_kv()|expandable_pointer())) - -> list(leveled_codec:ledger_kv()). + -> list(leveled_codec:ledger_kv()|expandable_pointer()). binaryslot_tolist(FullBin, PressMethod, IdxModDate, InitAcc) -> case crc_check_slot(FullBin) of {Header, Blocks} -> @@ -2426,7 +2605,7 @@ binaryslot_tolist(FullBin, PressMethod, IdxModDate, InitAcc) -> segment_check_fun(), list(leveled_codec:ledger_kv()|expandable_pointer()) ) -> - {list(leveled_codec:ledger_kv()), + {list(leveled_codec:ledger_kv()|expandable_pointer()), list({integer(), binary()})|none}. %% @doc %% Must return a trimmed and reversed list of results in the range @@ -2642,6 +2821,12 @@ block_offsetandlength(BlockLengths, BlockID) -> {B1L + B2L + B3L + B4L, B5L} end. +-spec fetch_value( + list(non_neg_integer()), + binary(), + binary(), + leveled_codec:ledger_key(), + press_method()) -> not_present|leveled_codec:ledger_kv(). fetch_value([], _BlockLengths, _Blocks, _Key, _PressMethod) -> not_present; fetch_value([Pos|Rest], BlockLengths, Blocks, Key, PressMethod) -> @@ -2791,12 +2976,12 @@ split_lists(KVList1, SlotLists, N, PressMethod, IdxModDate) -> split_lists(KVListRem, [SlotD|SlotLists], N - 1, PressMethod, IdxModDate). -spec merge_lists( - list(expanded_pointer()), - list(expanded_pointer()), + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), {boolean(), non_neg_integer()}, - sst_options(), boolean(), boolean()) -> - {list(expanded_pointer()), - list(expanded_pointer()), + sst_options(), boolean()) -> + {list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), list(binary_slot()), leveled_codec:ledger_key()|null, non_neg_integer()}. @@ -2805,9 +2990,7 @@ split_lists(KVList1, SlotLists, N, PressMethod, IdxModDate) -> %% provided may include pointers to fetch more Keys/Values from the source %% file merge_lists( - KVList1, KVList2, {IsBase, L}, SSTOpts, IndexModDate, SaveTombCount) -> - InitTombCount = - case SaveTombCount of true -> 0; false -> not_counted end, + KVList1, KVList2, {IsBase, L}, SSTOpts, IndexModDate) -> BuildTimings = case IsBase orelse lists:member(L, ?LOG_BUILDTIMINGS_LEVELS) of true -> @@ -2816,16 +2999,23 @@ merge_lists( no_timing end, merge_lists( - KVList1, KVList2, - {IsBase, L}, [], null, 0, - SSTOpts#sst_options.max_sstslots, SSTOpts#sst_options.press_method, - IndexModDate, InitTombCount, - BuildTimings). + KVList1, + KVList2, + {IsBase, L}, + [], + null, + 0, + SSTOpts#sst_options.max_sstslots, + SSTOpts#sst_options.press_method, + IndexModDate, + 0, + BuildTimings + ). -spec merge_lists( - list(expanded_pointer()), - list(expanded_pointer()), + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), {boolean(), non_neg_integer()}, list(binary_slot()), leveled_codec:ledger_key()|null, @@ -2833,11 +3023,11 @@ merge_lists( non_neg_integer(), press_method(), boolean(), - non_neg_integer()|not_counted, + non_neg_integer(), build_timings()) -> - {list(expanded_pointer()), list(expanded_pointer()), + {list(maybe_expanded_pointer()), list(maybe_expanded_pointer()), list(binary_slot()), leveled_codec:ledger_key()|null, - non_neg_integer()|not_counted}. + non_neg_integer()}. merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, MaxSlots, MaxSlots, _PressMethod, _IdxModDate, CountOfTombs, T0) -> @@ -2890,16 +3080,17 @@ merge_lists(KVL1, KVL2, LI, SlotList, FirstKey, SlotCount, MaxSlots, T2) end. --spec form_slot(list(expanded_pointer()), - list(expanded_pointer()), - {boolean(), non_neg_integer()}, - lookup|no_lookup, - non_neg_integer(), - list(leveled_codec:ledger_kv()), - leveled_codec:ledger_key()|null) -> - {list(expanded_pointer()), list(expanded_pointer()), - {lookup|no_lookup, list(leveled_codec:ledger_kv())}, - leveled_codec:ledger_key()}. +-spec form_slot( + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), + {boolean(), non_neg_integer()}, + lookup|no_lookup, + non_neg_integer(), + list(leveled_codec:ledger_kv()), + leveled_codec:ledger_key()|null) -> + {list(maybe_expanded_pointer()), list(maybe_expanded_pointer()), + {lookup|no_lookup, list(leveled_codec:ledger_kv())}, + leveled_codec:ledger_key()|null}. %% @doc %% Merge together Key Value lists to provide a reverse-ordered slot of KVs form_slot([], [], _LI, Type, _Size, Slot, FK) -> @@ -2932,7 +3123,7 @@ form_slot(KVList1, KVList2, LevelInfo, no_lookup, Size, Slot, FK) -> FK0); lookup -> case Size >= ?LOOK_SLOTSIZE of - true -> + true when FK =/= null -> {KVList1, KVList2, {no_lookup, Slot}, FK}; false -> form_slot( @@ -2950,22 +3141,28 @@ form_slot(KVList1, KVList2, LevelInfo, no_lookup, Size, Slot, FK) -> end. -spec key_dominates( - list(expanded_pointer()), - list(expanded_pointer()), - {boolean()|undefined, leveled_pmanifest:lsm_level()}) + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer()), + {boolean(), leveled_pmanifest:lsm_level()}) -> {{next_key, leveled_codec:ledger_kv()}|skipped_key, - list(expanded_pointer()), - list(expanded_pointer())}. + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())}. key_dominates([{pointer, SSTPid, Slot, StartKey, all}|T1], KL2, Level) -> key_dominates( expand_list_by_pointer( + % As the head is a pointer, the tail must be pointers too + % So eqwalizer is wrong that this may be + % [leveled_codec:ledger_kv()] + % eqwalizer:ignore {pointer, SSTPid, Slot, StartKey, all}, T1, ?MERGE_SCANWIDTH), KL2, Level); key_dominates([{next, ManEntry, StartKey}|T1], KL2, Level) -> key_dominates( expand_list_by_pointer( + % See above + % eqwalizer:ignore {next, ManEntry, StartKey, all}, T1, ?MERGE_SCANWIDTH), KL2, Level); @@ -2973,12 +3170,16 @@ key_dominates(KL1, [{pointer, SSTPid, Slot, StartKey, all}|T2], Level) -> key_dominates( KL1, expand_list_by_pointer( + % See above + % eqwalizer:ignore {pointer, SSTPid, Slot, StartKey, all}, T2, ?MERGE_SCANWIDTH), Level); key_dominates(KL1, [{next, ManEntry, StartKey}|T2], Level) -> key_dominates( KL1, expand_list_by_pointer( + % See above + % eqwalizer:ignore {next, ManEntry, StartKey, all}, T2, ?MERGE_SCANWIDTH), Level); key_dominates( @@ -2988,7 +3189,7 @@ key_dominates( [{K1, V1}|Rest1], [{K2, _V2}|_T2]=Rest2, {false, _TS}) when K1 < K2 -> {{next_key, {K1, V1}}, Rest1, Rest2}; key_dominates(KL1, KL2, Level) -> - case key_dominates_expanded(KL1, KL2) of + case key_dominates_comparison(KL1, KL2) of {{next_key, NKV}, Rest1, Rest2} -> case leveled_codec:maybe_reap_expiredkey(NKV, Level) of true -> @@ -3000,25 +3201,27 @@ key_dominates(KL1, KL2, Level) -> {skipped_key, Rest1, Rest2} end. --spec key_dominates_expanded( - list(expanded_pointer()), list(expanded_pointer())) +-spec key_dominates_comparison( + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())) + % first item in each list must be leveled_codec:ledger_kv() -> {{next_key, leveled_codec:ledger_kv()}|skipped_key, - list(expanded_pointer()), - list(expanded_pointer())}. -key_dominates_expanded([H1|T1], []) -> - {{next_key, H1}, T1, []}; -key_dominates_expanded([], [H2|T2]) -> - {{next_key, H2}, [], T2}; -key_dominates_expanded([{K1, _V1}|_T1]=LHL, [{K2, V2}|T2]) when K2 < K1 -> + list(maybe_expanded_pointer()), + list(maybe_expanded_pointer())}. +key_dominates_comparison([{K1, V1}|T1], []) -> + {{next_key, {K1, V1}}, T1, []}; +key_dominates_comparison([], [{K2, V2}|T2]) -> + {{next_key, {K2, V2}}, [], T2}; +key_dominates_comparison([{K1, _V1}|_T1]=LHL, [{K2, V2}|T2]) when K2 < K1 -> {{next_key, {K2, V2}}, LHL, T2}; -key_dominates_expanded([{K1, V1}|T1], [{K2, _V2}|_T2]=RHL) when K1 < K2 -> +key_dominates_comparison([{K1, V1}|T1], [{K2, _V2}|_T2]=RHL) when K1 < K2 -> {{next_key, {K1, V1}}, T1, RHL}; -key_dominates_expanded([H1|T1], [H2|T2]) -> - case leveled_codec:key_dominates(H1, H2) of +key_dominates_comparison([{K1, V1}|T1], [{K2, V2}|T2]) -> + case leveled_codec:key_dominates({K1, V1}, {K2, V2}) of true -> - {skipped_key, [H1|T1], T2}; + {skipped_key, [{K1, V1}|T1], T2}; false -> - {skipped_key, T1, [H2|T2]} + {skipped_key, T1, [{K2, V2}|T2]} end. @@ -3079,8 +3282,11 @@ log_buildtimings(Timings, LI) -> erlang:timestamp()|no_timing) -> ok. maybelog_fetch_timing(_Monitor, _Level, _Type, no_timing) -> ok; -maybelog_fetch_timing({Pid, _SlotFreq}, Level, Type, SW) -> - {TS1, _} = leveled_monitor:step_time(SW), +maybelog_fetch_timing({Pid, _SlotFreq}, Level, Type, SW) when is_pid(Pid), SW =/= no_timing -> + TS1 = + case leveled_monitor:step_time(SW) of + {TS, _NextSW} when is_integer(TS) -> TS + end, leveled_monitor:add_stat(Pid, {sst_fetch_update, Level, Type, TS1}). %%%============================================================================ @@ -3112,7 +3318,7 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) -> range_endpoint(), integer(), segment_check_fun(), - non_neg_integer()) -> list(leveled_codec:ledger_kv()|slot_pointer()). + non_neg_integer()) -> list(maybe_expanded_pointer()). %% @doc %% Get a range of {Key, Value} pairs as a list between StartKey and EndKey %% (inclusive). The ScanWidth is the maximum size of the range, a pointer @@ -3145,7 +3351,7 @@ testsst_new(RootPath, Filename, #sst_options{press_method=PressMethod, log_options=leveled_log:get_opts()}, sst_newmerge(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN, - OptsSST, false, false). + OptsSST, false). generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(Seqn, @@ -3157,16 +3363,21 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> Acc; generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> - BRand = leveled_rand:uniform(BRange), + BRand = rand:uniform(BRange), BNumber = lists:flatten(io_lib:format("B~6..0B", [BucketLow + BRand])), KNumber = - lists:flatten(io_lib:format("K~8..0B", [leveled_rand:uniform(1000000)])), - LK = leveled_codec:to_ledgerkey("Bucket" ++ BNumber, "Key" ++ KNumber, o), - Chunk = leveled_rand:rand_bytes(64), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, Seqn, Chunk, 64, infinity), + lists:flatten(io_lib:format("K~8..0B", [rand:uniform(1000000)])), + LK = + leveled_codec:to_objectkey( + list_to_binary("Bucket" ++ BNumber), + list_to_binary("Key" ++ KNumber), + o + ), + Chunk = crypto:strong_rand_bytes(64), + MV = leveled_codec:convert_to_ledgerv(LK, Seqn, Chunk, 64, infinity), MD = element(4, MV), + MD =/= null orelse error(bad_type), ?assertMatch(undefined, element(3, MD)), MD0 = [{magic_md, [<<0:32/integer>>, base64:encode(Chunk)]}], MV0 = setelement(4, MV, setelement(3, MD, MD0)), @@ -3176,14 +3387,13 @@ generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> BucketLow, BRange). - generate_indexkeys(Count) -> generate_indexkeys(Count, []). generate_indexkeys(0, IndexList) -> IndexList; generate_indexkeys(Count, IndexList) -> - Changes = generate_indexkey(leveled_rand:uniform(8000), Count), + Changes = generate_indexkey(rand:uniform(8000), Count), generate_indexkeys(Count - 1, IndexList ++ Changes). generate_indexkey(Term, Count) -> @@ -3255,7 +3465,7 @@ tombcount_tester(Level) -> KL2 = generate_indexkeys(N div 2), FlipToTombFun = fun({K, V}) -> - case leveled_rand:uniform(10) of + case rand:uniform(10) of X when X > 5 -> {K, setelement(2, V, tomb)}; _ -> @@ -3277,18 +3487,11 @@ tombcount_tester(Level) -> {RP, Filename} = {?TEST_AREA, "tombcount_test"}, OptsSST = - #sst_options{press_method=native, - log_options=leveled_log:get_opts()}, - {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, - N, OptsSST, false, false), - ?assertMatch(not_counted, sst_gettombcount(SST1)), - ok = sst_close(SST1), - ok = file:delete(filename:join(RP, Filename ++ ".sst")), - - {ok, SST2, _KD1, _BB1} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, Level, - N, OptsSST, false, true), + #sst_options{ + press_method=native, log_options=leveled_log:get_opts()}, + {ok, SST2, _KD1, _BB1} = + sst_newmerge( + RP, Filename, KVL1, KVL2, false, Level, N, OptsSST, true), ?assertMatch(ExpectedCount, sst_gettombcount(SST2)), ok = sst_close(SST2), @@ -3299,37 +3502,42 @@ form_slot_test() -> % If a skip key happens, mustn't switch to loookup by accident as could be % over the expected size SkippingKV = - {{o, "B1", "K9999", null}, {9999, tomb, {1234568, 1234567}, {}}}, + { + {o, <<"B1">>, <<"K9999">>, null}, + {9999, tomb, {1234568, 1234567}, {}} + }, Slot = - [{{o, "B1", "K5", null}, + [{{o, <<"B1">>, <<"K5">>, null}, {5, {active, infinity}, {99234568, 99234567}, {}}}], R1 = form_slot([SkippingKV], [], {true, 99999999}, no_lookup, ?LOOK_SLOTSIZE + 1, Slot, - {o, "B1", "K5", null}), - ?assertMatch({[], [], {no_lookup, Slot}, {o, "B1", "K5", null}}, R1). + {o, <<"B1">>, <<"K5">>, null}), + ?assertMatch( + {[], [], {no_lookup, Slot}, {o, <<"B1">>, <<"K5">>, null}}, + R1 + ). merge_tombstonelist_test() -> % Merge lists with nothing but tombstones, and file at basement level SkippingKV1 = - {{o, "B1", "K9995", null}, {9995, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9995">>, null}, {9995, tomb, {1234568, 1234567}, {}}}, SkippingKV2 = - {{o, "B1", "K9996", null}, {9996, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9996">>, null}, {9996, tomb, {1234568, 1234567}, {}}}, SkippingKV3 = - {{o, "B1", "K9997", null}, {9997, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9997">>, null}, {9997, tomb, {1234568, 1234567}, {}}}, SkippingKV4 = - {{o, "B1", "K9998", null}, {9998, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9998">>, null}, {9998, tomb, {1234568, 1234567}, {}}}, SkippingKV5 = - {{o, "B1", "K9999", null}, {9999, tomb, {1234568, 1234567}, {}}}, + {{o, <<"B1">>, <<"K9999">>, null}, {9999, tomb, {1234568, 1234567}, {}}}, R = merge_lists([SkippingKV1, SkippingKV3, SkippingKV5], [SkippingKV2, SkippingKV4], {true, 9999999}, #sst_options{press_method = native, max_sstslots = 256}, - ?INDEX_MODDATE, - true), + ?INDEX_MODDATE), ?assertMatch({[], [], [], null, 0}, R). @@ -3530,15 +3738,17 @@ indexed_list_mixedkeys_bitflip_test() -> TestKey2 = element(1, lists:nth(33, KVL1)), MH1 = leveled_codec:segment_hash(TestKey1), MH2 = leveled_codec:segment_hash(TestKey2), - + test_binary_slot(SlotBin, TestKey1, MH1, lists:nth(1, KVL1)), test_binary_slot(SlotBin, TestKey2, MH2, lists:nth(33, KVL1)), ToList = binaryslot_tolist(SlotBin, native, ?INDEX_MODDATE), ?assertMatch(Keys, ToList), - [Pos1] = find_pos(PosBin, segment_checker(extract_hash(MH1))), - [Pos2] = find_pos(PosBin, segment_checker(extract_hash(MH2))), + EH1 = case extract_hash(MH1) of Int1 when is_integer(Int1) -> Int1 end, + EH2 = case extract_hash(MH2) of Int2 when is_integer(Int2) -> Int2 end, + [Pos1] = find_pos(PosBin, segment_checker(EH1)), + [Pos2] = find_pos(PosBin, segment_checker(EH2)), {BN1, _BP1} = revert_position(Pos1), {BN2, _BP2} = revert_position(Pos2), {Offset1, Length1} = block_offsetandlength(Header, BN1), @@ -3593,7 +3803,7 @@ indexed_list_mixedkeys_bitflip_test() -> flip_byte(Binary, Offset, Length) -> - Byte1 = leveled_rand:uniform(Length) + Offset - 1, + Byte1 = rand:uniform(Length) + Offset - 1, <> = Binary, case A of 0 -> @@ -3639,15 +3849,16 @@ size_tester(KVL1, KVL2, N) -> io:format(user, "~nStarting ... test with ~w keys ~n", [N]), {RP, Filename} = {?TEST_AREA, "doublesize_test"}, - OptsSST = + Opts = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - {ok, SST1, _KD, _BB} = sst_newmerge(RP, Filename, - KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, - N, OptsSST, false, false), + {ok, SST1, _KD, _BB} = + sst_newmerge( + RP, Filename, KVL1, KVL2, false, ?DOUBLESIZE_LEVEL, N, Opts, false + ), ok = sst_close(SST1), {ok, SST2, _SKEK, Bloom} = - sst_open(RP, Filename ++ ".sst", OptsSST, ?DOUBLESIZE_LEVEL), + sst_open(RP, Filename ++ ".sst", Opts, ?DOUBLESIZE_LEVEL), FetchFun = fun({K, V}) -> {K0, V0} = sst_get(SST2, K), @@ -3696,8 +3907,20 @@ merge_tester(NewFunS, NewFunM) -> ?assertMatch(ExpFK2, FK2), ?assertMatch(ExpLK1, LK1), ?assertMatch(ExpLK2, LK2), - ML1 = [{next, #manifest_entry{owner = P1}, FK1}], - ML2 = [{next, #manifest_entry{owner = P2}, FK2}], + DSK = {o, <<"B">>, <<"SK">>, null}, + DEK = {o, <<"E">>, <<"EK">>, null}, + ML1 = + [{ + next, + leveled_pmanifest:new_entry(DSK, DEK, P1, "P1", none), + FK1 + }], + ML2 = + [{ + next, + leveled_pmanifest:new_entry(DSK, DEK, P2, "P2", none), + FK2 + }], NewR = NewFunM(?TEST_AREA, "level2_merge", ML1, ML2, false, 2, N * 2, native), {ok, P3, {{Rem1, Rem2}, FK3, LK3}, _Bloom3} = NewR, @@ -3795,9 +4018,14 @@ simple_persisted_rangesegfilter_tester(SSTNewFun) -> GetSegFun = fun(LK) -> - extract_hash( - leveled_codec:strip_to_segmentonly( - lists:keyfind(LK, 1, KVList1))) + case lists:keyfind(LK, 1, KVList1) of + LKV when LKV =/= false -> + extract_hash( + leveled_codec:strip_to_segmentonly( + LKV + ) + ) + end end, SegList = lists:map(GetSegFun, @@ -4042,15 +4270,18 @@ fetch_status_test() -> testsst_new(RP, Filename, 1, KVList1, length(KVList1), native), {status, Pid, {module, gen_statem}, SItemL} = sys:get_status(Pid), {data,[{"State", {reader, S}}]} = lists:nth(3, lists:nth(5, SItemL)), - true = is_integer(array:size(S#state.fetch_cache)), - true = is_integer(array:size(element(2, S#state.blockindex_cache))), + RS = S#state.read_state, + true = is_integer(array:size(RS#read_state.fetch_cache)), + true = is_integer(array:size(element(2, RS#read_state.blockindex_cache))), Status = format_status(#{reason => terminate, state => S}), ST = maps:get(state, Status), - ?assertMatch(redacted, ST#state.blockindex_cache), - ?assertMatch(redacted, ST#state.fetch_cache), + RST = ST#state.read_state, + ?assertMatch(redacted, RST#read_state.blockindex_cache), + ?assertMatch(redacted, RST#read_state.fetch_cache), NormStatus = format_status(#{reason => normal, state => S}), NST = maps:get(state, NormStatus), - ?assert(is_integer(array:size(NST#state.fetch_cache))), + RNST = NST#state.read_state, + ?assert(is_integer(array:size(RNST#read_state.fetch_cache))), ok = sst_close(Pid), ok = file:delete(filename:join(RP, Filename ++ ".sst")). @@ -4206,27 +4437,31 @@ check_binary_references(Pid) -> TotalBinMem. key_dominates_test() -> - KV1 = {{o, "Bucket", "Key1", null}, {5, {active, infinity}, 0, []}}, - KV2 = {{o, "Bucket", "Key3", null}, {6, {active, infinity}, 0, []}}, - KV3 = {{o, "Bucket", "Key2", null}, {3, {active, infinity}, 0, []}}, - KV4 = {{o, "Bucket", "Key4", null}, {7, {active, infinity}, 0, []}}, - KV5 = {{o, "Bucket", "Key1", null}, {4, {active, infinity}, 0, []}}, - KV6 = {{o, "Bucket", "Key1", null}, {99, {tomb, 999}, 0, []}}, - KV7 = {{o, "Bucket", "Key1", null}, {99, tomb, 0, []}}, + MakeKVFun = + fun(Key, SQN, Status) -> + {{o, <<"Bucket">>, Key, null}, {SQN, Status, 0, null, []}} + end, + KV1 = MakeKVFun(<<"Key1">>, 5, {active, infinity}), + KV2 = MakeKVFun(<<"Key3">>, 6, {active, infinity}), + KV3 = MakeKVFun(<<"Key2">>, 3, {active, infinity}), + KV4 = MakeKVFun(<<"Key4">>, 7, {active, infinity}), + KV5 = MakeKVFun(<<"Key1">>, 4, {active, infinity}), + KV6 = MakeKVFun(<<"Key1">>, 99, {tomb, 999}), + KV7 = MakeKVFun(<<"Key1">>, 99, tomb), KL1 = [KV1, KV2], KL2 = [KV3, KV4], ?assertMatch({{next_key, KV1}, [KV2], KL2}, - key_dominates(KL1, KL2, {undefined, 1})), + key_dominates(KL1, KL2, {false, 1})), ?assertMatch({{next_key, KV1}, KL2, [KV2]}, - key_dominates(KL2, KL1, {undefined, 1})), + key_dominates(KL2, KL1, {false, 1})), ?assertMatch({skipped_key, KL2, KL1}, - key_dominates([KV5|KL2], KL1, {undefined, 1})), + key_dominates([KV5|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV1}, [KV2], []}, - key_dominates(KL1, [], {undefined, 1})), + key_dominates(KL1, [], {false, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, - key_dominates([KV6|KL2], KL1, {undefined, 1})), + key_dominates([KV6|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV6}, KL2, [KV2]}, - key_dominates([KV6|KL2], [KV2], {undefined, 1})), + key_dominates([KV6|KL2], [KV2], {false, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, key_dominates([KV6|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, [KV6|KL2], [KV2]}, @@ -4248,9 +4483,9 @@ key_dominates_test() -> ?assertMatch({skipped_key, [], []}, key_dominates([], [KV7], {true, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, - key_dominates([KV7|KL2], KL1, {undefined, 1})), + key_dominates([KV7|KL2], KL1, {false, 1})), ?assertMatch({{next_key, KV7}, KL2, [KV2]}, - key_dominates([KV7|KL2], [KV2], {undefined, 1})), + key_dominates([KV7|KL2], [KV2], {false, 1})), ?assertMatch({skipped_key, [KV7|KL2], [KV2]}, key_dominates([KV7|KL2], KL1, {true, 1})), ?assertMatch({skipped_key, KL2, [KV2]}, @@ -4277,11 +4512,9 @@ hashmatching_bytreesize_test() -> B, list_to_binary("Key" ++ integer_to_list(X)), null}, - LKV = leveled_codec:generate_ledgerkv(LK, - X, - V, - byte_size(V), - {active, infinity}), + LKV = + leveled_codec:generate_ledgerkv( + LK, X, V, byte_size(V), infinity), {_Bucket, _Key, MetaValue, _Hashes, _LastMods} = LKV, {LK, MetaValue} end, @@ -4340,8 +4573,8 @@ corrupted_block_rangetester(PressMethod, TestCount) -> KVL1 = lists:ukeysort(1, generate_randomkeys(1, N, 1, 2)), RandomRangesFun = fun(_X) -> - SKint = leveled_rand:uniform(90) + 1, - EKint = min(N, leveled_rand:uniform(N - SKint)), + SKint = rand:uniform(90) + 1, + EKint = min(N, rand:uniform(N - SKint)), SK = element(1, lists:nth(SKint, KVL1)), EK = element(1, lists:nth(EKint, KVL1)), {SK, EK} @@ -4354,7 +4587,7 @@ corrupted_block_rangetester(PressMethod, TestCount) -> B5 = serialise_block(lists:sublist(KVL1, 81, 20), PressMethod), CorruptBlockFun = fun(Block) -> - case leveled_rand:uniform(10) < 2 of + case rand:uniform(10) < 2 of true -> flip_byte(Block, 0 , byte_size(Block)); false -> @@ -4408,18 +4641,20 @@ corrupted_block_fetch_tester(PressMethod) -> CheckFun = fun(N, {AccHit, AccMiss}) -> - PosL = [min(0, leveled_rand:uniform(N - 2)), N - 1], + PosL = [min(0, rand:uniform(N - 2)), N - 1], {LK, LV} = lists:nth(N, KVL1), {BlockLengths, 0, PosBinIndex} = extract_header(Header, false), - R = check_blocks(PosL, - CorruptSlotBin, - BlockLengths, - byte_size(PosBinIndex), - LK, - PressMethod, - false, - not_present), + R = + check_blocks_matchkey( + PosL, + CorruptSlotBin, + BlockLengths, + byte_size(PosBinIndex), + LK, + PressMethod, + false + ), case R of not_present -> {AccHit, AccMiss + 1}; @@ -5004,7 +5239,11 @@ range_key_lestthanprefix_test() -> ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")). size_summary(P) -> - Summary = element(2, element(2, sys:get_state(P))), + Summary = + case sys:get_state(P) of + State when is_tuple(State) -> + element(2, element(2, State)) + end, true = is_record(Summary, summary), erts_debug:flat_size(Summary). @@ -5016,14 +5255,12 @@ print_compare_size(Type, OptimisedSize, UnoptimisedSize) -> % Reduced by at least a quarter ?assert(OptimisedSize < (UnoptimisedSize - (UnoptimisedSize div 4))). - single_key_test() -> FileName = "single_key_test", Field = <<"t1_bin">>, - LK = leveled_codec:to_ledgerkey(<<"Bucket0">>, <<"Key0">>, ?STD_TAG), - Chunk = leveled_rand:rand_bytes(16), - {_B, _K, MV, _H, _LMs} = - leveled_codec:generate_ledgerkv(LK, 1, Chunk, 16, infinity), + LK = leveled_codec:to_objectkey(<<"Bucket0">>, <<"Key0">>, ?STD_TAG), + Chunk = crypto:strong_rand_bytes(16), + MV = leveled_codec:convert_to_ledgerv(LK, 1, Chunk, 16, infinity), OptsSST = #sst_options{press_method=native, log_options=leveled_log:get_opts()}, @@ -5075,14 +5312,14 @@ strange_range_test() -> #sst_options{press_method=native, log_options=leveled_log:get_opts()}, - FK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0">>, ?RIAK_TAG), - LK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K02">>, ?RIAK_TAG), - EK = leveled_codec:to_ledgerkey({<<"T0">>, <<"B0">>}, <<"K0299">>, ?RIAK_TAG), + FK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K0">>, ?RIAK_TAG), + LK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K02">>, ?RIAK_TAG), + EK = leveled_codec:to_objectkey({<<"T0">>, <<"B0">>}, <<"K0299">>, ?RIAK_TAG), KL1 = lists:map( fun(I) -> - leveled_codec:to_ledgerkey( + leveled_codec:to_objectkey( {<<"T0">>, <<"B0">>}, list_to_binary("K00" ++ integer_to_list(I)), ?RIAK_TAG) @@ -5091,7 +5328,7 @@ strange_range_test() -> KL2 = lists:map( fun(I) -> - leveled_codec:to_ledgerkey( + leveled_codec:to_objectkey( {<<"T0">>, <<"B0">>}, list_to_binary("K02" ++ integer_to_list(I)), ?RIAK_TAG) @@ -5114,8 +5351,8 @@ strange_range_test() -> {ok, P1, {FK, EK}, _Bloom1} = sst_new(?TEST_AREA, FileName, 1, KVL, 6000, OptsSST), - ?assertMatch(LK, element(1, sst_get(P1, LK))), - ?assertMatch(FK, element(1, sst_get(P1, FK))), + ?assertMatch({LK, _}, sst_get(P1, LK)), + ?assertMatch({FK, _}, sst_get(P1, FK)), ok = sst_close(P1), ok = file:delete(filename:join(?TEST_AREA, FileName ++ ".sst")), @@ -5171,7 +5408,7 @@ start_sst_fun(ProcessToInform) -> blocks_required_test() -> B = <<"Bucket">>, Idx = <<"idx_bin">>, - Chunk = leveled_rand:rand_bytes(32), + Chunk = crypto:strong_rand_bytes(32), KeyFun = fun(I) -> list_to_binary(io_lib:format("B~6..0B", [I])) @@ -5194,7 +5431,7 @@ blocks_required_test() -> element( 3, leveled_codec:generate_ledgerkv( - IdxKey(I), I, null, 0, infinity)) + IdxKey(I), I, <<>>, 0, infinity)) end, Block1L = lists:map(fun(I) -> {IdxKey(I), IdxValue(I)} end, lists:seq(1, 16)), diff --git a/src/leveled_tictac.erl b/src/leveled_tictac.erl index 5eb201d6..6bdfc33e 100644 --- a/src/leveled_tictac.erl +++ b/src/leveled_tictac.erl @@ -101,7 +101,7 @@ width :: integer(), segment_count :: integer(), level1 :: level1_map(), - level2 :: array:array() + level2 :: array:array(binary()) }). -type level1_map() :: #{non_neg_integer() => binary()}|binary(). @@ -161,6 +161,8 @@ new_tree(TreeID, Size, UseMap) -> width = Width, segment_count = Width * ?L2_CHUNKSIZE, level1 = Lv1Init, + % array values are indeed all binaries + % eqwalizer:ignore level2 = Lv2Init }. @@ -196,13 +198,16 @@ import_tree(ExportedTree) -> [{<<"level1">>, L1Base64}, {<<"level2">>, {struct, L2List}}]} = ExportedTree, L1Bin = base64:decode(L1Base64), - Sizes = lists:map(fun(SizeTag) -> {SizeTag, get_size(SizeTag)} end, - ?VALID_SIZES), + Sizes = + lists:map( + fun(SizeTag) -> {SizeTag, get_size(SizeTag)} end, + ?VALID_SIZES + ), Width = byte_size(L1Bin) div ?HASH_SIZE, {Size, _Width} = lists:keyfind(Width, 2, Sizes), %% assert that side is indeed the provided width true = get_size(Size) == Width, - Lv2Init = array:new([{size, Width}]), + Lv2Init = array:new([{size, Width}, {default, ?EMPTY}]), FoldFun = fun({X, EncodedL2SegBin}, L2Array) -> L2SegBin = zlib:uncompress(base64:decode(EncodedL2SegBin)), @@ -216,6 +221,8 @@ import_tree(ExportedTree) -> width = Width, segment_count = Width * ?L2_CHUNKSIZE, level1 = to_level1_map(L1Bin), + % array values are indeed all binaries + % eqwalizer:ignore level2 = Lv2 }. @@ -229,12 +236,14 @@ import_tree(ExportedTree) -> add_kv(TicTacTree, Key, Value, BinExtractFun) -> add_kv(TicTacTree, Key, Value, BinExtractFun, false). --spec add_kv( - tictactree(), term(), term(), bin_extract_fun(), boolean()) - -> tictactree()|{tictactree(), integer()}. +-spec add_kv + (tictactree(), term(), term(), bin_extract_fun(), true) -> + {tictactree(), integer()}; + (tictactree(), term(), term(), bin_extract_fun(), false) -> + tictactree(). %% @doc %% add_kv with ability to return segment ID of Key added -add_kv(TicTacTree, Key, Value, BinExtractFun, ReturnSegment) -> +add_kv(TicTacTree, Key, Value, BinExtractFun, true) -> {BinK, BinV} = BinExtractFun(Key, Value), {SegHash, SegChangeHash} = tictac_hash(BinK, BinV), Segment = get_segment(SegHash, TicTacTree#tictactree.segment_count), @@ -249,12 +258,9 @@ add_kv(TicTacTree, Key, Value, BinExtractFun, ReturnSegment) -> replace_segment( SegLeaf1Upd, SegLeaf2Upd, L1Extract, L2Extract, TicTacTree ), - case ReturnSegment of - true -> - {UpdatedTree, Segment}; - false -> - UpdatedTree - end. + {UpdatedTree, Segment}; +add_kv(TicTacTree, Key, Value, BinExtractFun, false) -> + element(1, add_kv(TicTacTree, Key, Value, BinExtractFun, true)). -spec alter_segment(integer(), integer(), tictactree()) -> tictactree(). %% @doc @@ -373,16 +379,13 @@ get_segment(Hash, TreeSize) -> %% has already been taken. If the value is not a pre-extracted hash just use %% erlang:phash2. If an exportable hash of the value is required this should %% be managed through the add_kv ExtractFun providing a pre-prepared Hash. -tictac_hash(BinKey, Val) when is_binary(BinKey) -> +tictac_hash( + BinKey, {is_hash, HashedVal}) + when is_binary(BinKey), is_integer(HashedVal) -> {HashKeyToSeg, AltHashKey} = keyto_doublesegment32(BinKey), - HashVal = - case Val of - {is_hash, HashedVal} -> - HashedVal; - _ -> - erlang:phash2(Val) - end, - {HashKeyToSeg, AltHashKey bxor HashVal}. + {HashKeyToSeg, AltHashKey bxor HashedVal}; +tictac_hash(BinKey, ValToHash) when is_binary(BinKey) -> + tictac_hash(BinKey, {is_hash, erlang:phash2(ValToHash)}). -spec keyto_doublesegment32( binary()) -> {non_neg_integer(), non_neg_integer()}. @@ -961,7 +964,7 @@ timing_test() -> timing_tester(KeyCount, SegCount, SmallSize, LargeSize) -> SegList = lists:map(fun(_C) -> - leveled_rand:uniform(get_size(SmallSize) * ?L2_CHUNKSIZE - 1) + rand:uniform(get_size(SmallSize) * ?L2_CHUNKSIZE - 1) end, lists:seq(1, SegCount)), KeyToSegFun = diff --git a/src/leveled_tree.erl b/src/leveled_tree.erl index d824c9f8..15cfde26 100644 --- a/src/leveled_tree.erl +++ b/src/leveled_tree.erl @@ -29,9 +29,7 @@ -define(SKIP_WIDTH, 16). -type tree_type() :: tree|idxt|skpl. --type leveled_tree() :: {tree_type(), - integer(), % length - any()}. +-type leveled_tree() :: {tree_type(), integer(), any()}. -export_type([leveled_tree/0]). @@ -104,7 +102,7 @@ match(Key, {tree, _L, Tree}) -> {_NK, SL, _Iter} -> lookup_match(Key, SL) end; -match(Key, {idxt, _L, {TLI, IDX}}) -> +match(Key, {idxt, _L, {TLI, IDX}}) when is_tuple(TLI) -> Iter = tree_iterator_from(Key, IDX), case tree_next(Iter) of none -> @@ -136,7 +134,7 @@ search(Key, {tree, _L, Tree}, StartKeyFun) -> {K, V} end end; -search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) -> +search(Key, {idxt, _L, {TLI, IDX}}, StartKeyFun) when is_tuple(TLI) -> Iter = tree_iterator_from(Key, IDX), case tree_next(Iter) of none -> @@ -235,14 +233,13 @@ to_list({tree, _L, Tree}) -> Acc ++ SL end, lists:foldl(FoldFun, [], tree_to_list(Tree)); -to_list({idxt, _L, {TLI, _IDX}}) -> +to_list({idxt, _L, {TLI, _IDX}}) when is_tuple(TLI) -> lists:append(tuple_to_list(TLI)); -to_list({skpl, _L, SkipList}) -> +to_list({skpl, _L, SkipList}) when is_list(SkipList) -> FoldFun = fun({_M, SL}, Acc) -> [SL|Acc] end, - Lv1List = lists:reverse(lists:foldl(FoldFun, [], SkipList)), Lv0List = lists:reverse(lists:foldl(FoldFun, [], lists:append(Lv1List))), lists:append(Lv0List). @@ -580,13 +577,13 @@ generate_randomkeys(Seqn, Count, BucketRangeLow, BucketRangeHigh) -> generate_randomkeys(_Seqn, 0, Acc, _BucketLow, _BucketHigh) -> Acc; generate_randomkeys(Seqn, Count, Acc, BucketLow, BRange) -> - BRand = leveled_rand:uniform(BRange), + BRand = rand:uniform(BRange), BNumber = lists:flatten( io_lib:format("K~4..0B", [BucketLow + BRand])), KNumber = lists:flatten( - io_lib:format("K~8..0B", [leveled_rand:uniform(1000)])), + io_lib:format("K~8..0B", [rand:uniform(1000)])), {K, V} = {{o_kv, {<<"btype">>, list_to_binary("Bucket" ++ BNumber)}, @@ -608,7 +605,7 @@ generate_simplekeys(Seqn, Count, Acc) -> KNumber = list_to_binary( lists:flatten( - io_lib:format("K~8..0B", [leveled_rand:uniform(100000)]))), + io_lib:format("K~8..0B", [rand:uniform(100000)]))), generate_simplekeys(Seqn + 1, Count - 1, [{KNumber, Seqn}|Acc]). @@ -958,14 +955,13 @@ search_range_idx_test() -> {o_rkv,"Bucket1","Key1",null}, "<0.320.0>","./16_1_6.sst", none}}]}, {1,{{o_rkv,"Bucket1","Key1",null},1,nil,nil}}}}, - StartKeyFun = - fun(ME) -> - ME#manifest_entry.start_key - end, - R = search_range({o_rkv, "Bucket", null, null}, - {o_rkv, "Bucket", null, null}, - Tree, - StartKeyFun), + R = + search_range( + {o_rkv, "Bucket", null, null}, + {o_rkv, "Bucket", null, null}, + Tree, + fun leveled_pmanifest:entry_startkey/1 + ), ?assertMatch(1, length(R)). -endif. diff --git a/src/leveled_util.erl b/src/leveled_util.erl index ac8c4390..630e9f42 100644 --- a/src/leveled_util.erl +++ b/src/leveled_util.erl @@ -23,7 +23,7 @@ %% Credit to %% https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl generate_uuid() -> - <> = leveled_rand:rand_bytes(16), + <> = crypto:strong_rand_bytes(16), L = io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]), binary_to_list(list_to_binary(L)). diff --git a/test/end_to_end/appdefined_SUITE.erl b/test/end_to_end/appdefined_SUITE.erl index 4c508a0a..906249ac 100644 --- a/test/end_to_end/appdefined_SUITE.erl +++ b/test/end_to_end/appdefined_SUITE.erl @@ -73,7 +73,7 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> [{bespoke_tag1, retain}, {bespoke_tag2, retain}]}, {override_functions, Functions}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - Value = leveled_rand:rand_bytes(512), + Value = crypto:strong_rand_bytes(512), MapFun = fun(C) -> {C, object_generator(C, Value)} @@ -119,7 +119,7 @@ application_defined_tag_tester(KeyCount, Tag, Functions, ExpectMD) -> object_generator(Count, V) -> Hash = erlang:phash2({count, V}), - Random = leveled_rand:uniform(1000), + Random = rand:uniform(1000), Key = list_to_binary(leveled_util:generate_uuid()), Bucket = <<"B">>, {Bucket, diff --git a/test/end_to_end/basic_SUITE.erl b/test/end_to_end/basic_SUITE.erl index 22c7b916..13beb8d4 100644 --- a/test/end_to_end/basic_SUITE.erl +++ b/test/end_to_end/basic_SUITE.erl @@ -55,15 +55,19 @@ simple_put_fetch_head_delete(_Config) -> simple_test_withlog(LogLevel, ForcedLogs) -> RootPath = testutil:reset_filestructure(), - StartOpts1 = [{root_path, RootPath}, - {sync_strategy, testutil:sync_strategy()}, - {log_level, LogLevel}, - {forced_logs, ForcedLogs}], + StartOpts1 = + [ + {root_path, RootPath}, + {sync_strategy, testutil:sync_strategy()}, + {log_level, LogLevel}, + {forced_logs, ForcedLogs}, + {max_pencillercachesize, 200} + ], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), {TestObject, TestSpec} = testutil:generate_testobject(), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), - testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie1, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie1), StartOpts2 = [{root_path, RootPath}, {max_journalsize, 3000000}, @@ -78,29 +82,49 @@ simple_test_withlog(LogLevel, ForcedLogs) -> ChkList1 = lists:sublist(lists:sort(ObjList1), 100), testutil:check_forlist(Bookie2, ChkList1), testutil:check_forobject(Bookie2, TestObject), - testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), - ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", "Value2", - [{add, "Index1", "Term1"}]), - {ok, "Value2"} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), - {ok, {62888926, S, undefined}} = - leveled_bookie:book_head(Bookie2, "Bucket1", "Key2"), - true = (S == 58) or (S == 60), + testutil:check_formissingobject(Bookie2, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_put( + Bookie2, + <<"Bucket1">>, + <<"Key2">>, + <<"Value2">>, + [{add, <<"Index1">>, <<"Term1">>}] + ), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie2, <<"Bucket1">>, <<"Key2">>), + {ok, {2220864, S, undefined}} = + leveled_bookie:book_head(Bookie2, <<"Bucket1">>, <<"Key2">>), + true = (S == 63) or (S == 65), % After OTP 26 the object is 58 bytes not 60 - testutil:check_formissingobject(Bookie2, "Bucket1", "Key2"), - ok = leveled_bookie:book_put(Bookie2, "Bucket1", "Key2", <<"Value2">>, - [{remove, "Index1", "Term1"}, - {add, "Index1", <<"Term2">>}]), - {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie2, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie2, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_put( + Bookie2, + <<"Bucket1">>, + <<"Key2">>, + <<"Value2">>, + [{remove, <<"Index1">>, <<"Term1">>}, + {add, <<"Index1">>, <<"Term2">>}] + ), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie2, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie2), {ok, Bookie3} = leveled_bookie:book_start(StartOpts2), - {ok, <<"Value2">>} = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), - ok = leveled_bookie:book_delete(Bookie3, "Bucket1", "Key2", - [{remove, "Index1", "Term1"}]), - not_found = leveled_bookie:book_get(Bookie3, "Bucket1", "Key2"), - not_found = leveled_bookie:book_head(Bookie3, "Bucket1", "Key2"), + {ok, <<"Value2">>} = + leveled_bookie:book_get(Bookie3, <<"Bucket1">>, <<"Key2">>), + ok = + leveled_bookie:book_delete( + Bookie3, + <<"Bucket1">>, + <<"Key2">>, + [{remove, <<"Index1">>, <<"Term1">>}] + ), + not_found = leveled_bookie:book_get(Bookie3, <<"Bucket1">>, <<"Key2">>), + not_found = leveled_bookie:book_head(Bookie3, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_close(Bookie3), {ok, Bookie4} = leveled_bookie:book_start(StartOpts2), - not_found = leveled_bookie:book_get(Bookie4, "Bucket1", "Key2"), + not_found = leveled_bookie:book_get(Bookie4, <<"Bucket1">>, <<"Key2">>), ok = leveled_bookie:book_destroy(Bookie4). many_put_fetch_head(_Config) -> @@ -168,7 +192,7 @@ many_put_fetch_head(_Config) -> not_found = leveled_bookie:book_sqn(Bookie3, testutil:get_bucket(TestObject), testutil:get_key(TestObject)), - testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"), + testutil:check_formissingobject(Bookie3, <<"Bookie1">>, <<"MissingKey0123">>), ok = leveled_bookie:book_destroy(Bookie3). bigjournal_littlejournal(_Config) -> @@ -181,7 +205,7 @@ bigjournal_littlejournal(_Config) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ObjL1 = testutil:generate_objects(100, 1, [], - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), fun() -> [] end, <<"B">>), testutil:riakload(Bookie1, ObjL1), ok = leveled_bookie:book_close(Bookie1), @@ -189,7 +213,7 @@ bigjournal_littlejournal(_Config) -> {ok, Bookie2} = leveled_bookie:book_start(StartOpts2), ObjL2 = testutil:generate_objects(10, 1000, [], - leveled_rand:rand_bytes(10000), + crypto:strong_rand_bytes(10000), fun() -> [] end, <<"B">>), testutil:riakload(Bookie2, ObjL2), testutil:check_forlist(Bookie2, ObjL1), @@ -214,7 +238,7 @@ bigsst_littlesst(_Config) -> 100000, 1, [], - leveled_rand:rand_bytes(100), + crypto:strong_rand_bytes(100), fun() -> [] end, <<"B">>) ), @@ -260,13 +284,16 @@ journal_compaction_tester(Restart, WRP) -> ChkList1 = lists:sublist(lists:sort(ObjList1), 10000), testutil:check_forlist(Bookie0, ChkList1), testutil:check_forobject(Bookie0, TestObject), - {B2, K2, V2, Spec2, MD} = {"Bucket2", - "Key2", - "Value2", - [], - [{"MDK2", "MDV2"}]}, - {TestObject2, TestSpec2} = testutil:generate_testobject(B2, K2, - V2, Spec2, MD), + {B2, K2, V2, Spec2, MD} = + { + <<"Bucket2">>, + <<"Key2">>, + <<"Value2">>, + [], + [{<<"MDK2">>, <<"MDV2">>}] + }, + {TestObject2, TestSpec2} = + testutil:generate_testobject(B2, K2, V2, Spec2, MD), ok = testutil:book_riakput(Bookie0, TestObject2, TestSpec2), ok = leveled_bookie:book_compactjournal(Bookie0, 30000), testutil:check_forlist(Bookie0, ChkList1), @@ -277,13 +304,15 @@ journal_compaction_tester(Restart, WRP) -> testutil:check_forobject(Bookie0, TestObject2), %% Delete some of the objects ObjListD = testutil:generate_objects(10000, 2), - lists:foreach(fun({_R, O, _S}) -> - testutil:book_riakdelete(Bookie0, - testutil:get_bucket(O), - testutil:get_key(O), - []) - end, - ObjListD), + lists:foreach( + fun({_R, O, _S}) -> + testutil:book_riakdelete(Bookie0, + testutil:get_bucket(O), + testutil:get_key(O), + []) + end, + ObjListD + ), %% Now replace all the other objects ObjList2 = testutil:generate_objects(40000, 10002), @@ -539,11 +568,11 @@ fetchput_snapshot(_Config) -> % smaller due to replacements and files deleting % This is dependent on the sleep though (yuk) - {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, <<"Bucket1">>), true = B1Size > 0, true = B1Count == 1, - {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, "Bucket1"), - {BSize, BCount} = testutil:check_bucket_stats(Bookie2, "Bucket"), + {B1Size, B1Count} = testutil:check_bucket_stats(Bookie2, <<"Bucket1">>), + {BSize, BCount} = testutil:check_bucket_stats(Bookie2, <<"Bucket">>), true = BSize > 0, true = BCount == 180000, @@ -622,82 +651,78 @@ load_and_count(JournalSize, BookiesMemSize, PencillerMemSize) -> testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), G1 = fun testutil:generate_smallobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Loading larger compressible objects~n"), G2 = fun testutil:generate_compressibleobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G2), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 100000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G2), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Replacing small objects~n"), - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Count == 200000 -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Count == 200000 -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), io:format("Loading more small objects~n"), io:format("Now with unused snapshot so deletions are blocked~n"), {ok, PclClone, null} = leveled_bookie:book_snapshot(Bookie1, ledger, undefined, true), - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G2), - {_S, Count} = - testutil:check_bucket_stats(Bookie1, "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 200000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G2), + {_S, Count} = + testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 200000, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), ok = leveled_penciller:pcl_close(PclClone), - {_S, 300000} = testutil:check_bucket_stats(Bookie1, "Bucket"), + {_S, 300000} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), ok = leveled_bookie:book_close(Bookie1), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), - {_, 300000} = testutil:check_bucket_stats(Bookie2, "Bucket"), + {_, 300000} = testutil:check_bucket_stats(Bookie2, <<"Bucket">>), ok = leveled_bookie:book_close(Bookie2), @@ -722,21 +747,19 @@ load_and_count_withdelete(_Config) -> testutil:check_forobject(Bookie1, TestObject), io:format("Loading initial small objects~n"), G1 = fun testutil:generate_smallobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - TestObject, - G1), - {_S, Count} = testutil:check_bucket_stats(Bookie1, - "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 0, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, TestObject, G1), + {_S, Count} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 0, + lists:seq(1, 20) + ), testutil:check_forobject(Bookie1, TestObject), {BucketD, KeyD} = {testutil:get_bucket(TestObject), testutil:get_key(TestObject)}, @@ -746,21 +769,19 @@ load_and_count_withdelete(_Config) -> {_, 0} = testutil:check_bucket_stats(Bookie1, BucketD), io:format("Loading larger compressible objects~n"), G2 = fun testutil:generate_compressibleobjects/2, - lists:foldl(fun(_X, Acc) -> - testutil:load_objects(5000, - [Acc + 2], - Bookie1, - no_check, - G2), - {_S, Count} = testutil:check_bucket_stats(Bookie1, - "Bucket"), - if - Acc + 5000 == Count -> - ok - end, - Acc + 5000 end, - 100000, - lists:seq(1, 20)), + lists:foldl( + fun(_X, Acc) -> + testutil:load_objects( + 5000, [Acc + 2], Bookie1, no_check, G2), + {_S, Count} = testutil:check_bucket_stats(Bookie1, <<"Bucket">>), + if + Acc + 5000 == Count -> + ok + end, + Acc + 5000 end, + 100000, + lists:seq(1, 20) + ), not_found = testutil:book_riakget(Bookie1, BucketD, KeyD), ok = leveled_bookie:book_close(Bookie1), @@ -780,11 +801,8 @@ space_clear_ondelete(_Config) -> {sync_strategy, testutil:sync_strategy()}], {ok, Book1} = leveled_bookie:book_start(StartOpts1), G2 = fun testutil:generate_compressibleobjects/2, - testutil:load_objects(20000, - [uuid, uuid, uuid, uuid], - Book1, - no_check, - G2), + testutil:load_objects( + 20000, [uuid, uuid, uuid, uuid], Book1, no_check, G2), FoldKeysFun = fun(B, K, Acc) -> [{B, K}|Acc] end, @@ -808,10 +826,9 @@ space_clear_ondelete(_Config) -> FoldObjectsFun = fun(B, K, ObjBin, Acc) -> [{B, K, erlang:phash2(ObjBin)}|Acc] end, - {async, HTreeF1} = leveled_bookie:book_objectfold(Book1, - ?RIAK_TAG, - {FoldObjectsFun, []}, - false), + {async, HTreeF1} = + leveled_bookie:book_objectfold( + Book1, ?RIAK_TAG, {FoldObjectsFun, []}, false), % This query does not Snap PreFold - and so will not prevent % pending deletes from prompting actual deletes @@ -822,32 +839,34 @@ space_clear_ondelete(_Config) -> % Delete the keys SW2 = os:timestamp(), - lists:foreach(fun({Bucket, Key}) -> - testutil:book_riakdelete(Book1, - Bucket, - Key, - []) - end, - KL1), - io:format("Deletion took ~w microseconds for 80K keys~n", - [timer:now_diff(os:timestamp(), SW2)]), - - + lists:foreach( + fun({Bucket, Key}) -> + testutil:book_riakdelete(Book1, Bucket, Key, []) + end, + KL1), + io:format( + "Deletion took ~w microseconds for 80K keys~n", + [timer:now_diff(os:timestamp(), SW2)]), ok = leveled_bookie:book_compactjournal(Book1, 30000), F = fun leveled_bookie:book_islastcompactionpending/1, - lists:foldl(fun(X, Pending) -> - case Pending of - false -> - false; - true -> - io:format("Loop ~w waiting for journal " - ++ "compaction to complete~n", [X]), - timer:sleep(20000), - F(Book1) - end end, - true, - lists:seq(1, 15)), + lists:foldl( + fun(X, Pending) -> + case Pending of + false -> + false; + true -> + io:format( + "Loop ~w waiting for journal " + "compaction to complete~n", + [X] + ), + timer:sleep(20000), + F(Book1) + end + end, + true, + lists:seq(1, 15)), io:format("Waiting for journal deletes - blocked~n"), timer:sleep(20000), @@ -1113,7 +1132,7 @@ many_put_fetch_switchcompression_tester(CompressionMethod) -> %% Change method back again {ok, Bookie3} = leveled_bookie:book_start(StartOpts1), - testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"), + testutil:check_formissingobject(Bookie3, <<"Bookie1">>, "MissingKey0123"), lists:foreach( fun(CL) -> ok = testutil:check_forlist(Bookie3, CL) end, CL2s), lists:foreach( @@ -1244,10 +1263,12 @@ bigpcl_bucketlist(_Config) -> MapFun = fun(B) -> - testutil:generate_objects(ObjectCount, 1, [], - leveled_rand:rand_bytes(100), - fun() -> [] end, - B) + testutil:generate_objects( + ObjectCount, 1, [], + crypto:strong_rand_bytes(100), + fun() -> [] end, + B + ) end, ObjLofL = lists:map(MapFun, BucketList), lists:foreach(fun(ObjL) -> testutil:riakload(Bookie1, ObjL) end, ObjLofL), @@ -1263,11 +1284,15 @@ bigpcl_bucketlist(_Config) -> FBAccT = {BucketFold, sets:new()}, {async, BucketFolder1} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, BucketList}, - FBAccT, - false, false, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, + false, + false + ), {FoldTime1, BucketList1} = timer:tc(BucketFolder1, []), true = BucketCount == sets:size(BucketList1), @@ -1276,11 +1301,15 @@ bigpcl_bucketlist(_Config) -> {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), {async, BucketFolder2} = - leveled_bookie:book_headfold(Bookie2, - ?RIAK_TAG, - {bucket_list, BucketList}, - FBAccT, - false, false, false), + leveled_bookie:book_headfold( + Bookie2, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, + false, + false + ), {FoldTime2, BucketList2} = timer:tc(BucketFolder2, []), true = BucketCount == sets:size(BucketList2), diff --git a/test/end_to_end/iterator_SUITE.erl b/test/end_to_end/iterator_SUITE.erl index 15236f61..855aad8e 100644 --- a/test/end_to_end/iterator_SUITE.erl +++ b/test/end_to_end/iterator_SUITE.erl @@ -57,7 +57,7 @@ expiring_indexes(_Config) -> Indexes9 = testutil:get_randomindexes_generator(2), TempRiakObjects = testutil:generate_objects( - KeyCount, binary_uuid, [], V9, Indexes9, "riakBucket"), + KeyCount, binary_uuid, [], V9, Indexes9, <<"riakBucket">>), IBKL1 = testutil:stdload_expiring(Bookie1, KeyCount, Future), lists:foreach( @@ -147,11 +147,13 @@ expiring_indexes(_Config) -> Bookie1, B0, K0, 5, <<"value">>, leveled_util:integer_now() + 10), timer:sleep(1000), {async, Folder2} = IndexFold(), - leveled_bookie:book_indexfold(Bookie1, - B0, - {FoldFun, InitAcc}, - {<<"temp_int">>, 5, 8}, - {true, undefined}), + leveled_bookie:book_indexfold( + Bookie1, + B0, + {FoldFun, InitAcc}, + {<<"temp_int">>, 5, 8}, + {true, undefined} + ), QR2 = Folder2(), io:format("Query with additional entry length ~w~n", [length(QR2)]), true = lists:sort(QR2) == lists:sort([{5, B0, K0}|LoadedEntriesInRange]), @@ -208,11 +210,9 @@ breaking_folds(_Config) -> {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = testutil:get_randomindexes_generator(8), - ObjL1 = testutil:generate_objects(KeyCount, - binary_uuid, - [], - ObjectGen, - IndexGen), + ObjL1 = + testutil:generate_objects( + KeyCount, binary_uuid, [], ObjectGen, IndexGen), testutil:riakload(Bookie1, ObjL1), % Find all keys index, and then same again but stop at a midpoint using a @@ -261,7 +261,6 @@ breaking_folds(_Config) -> io:format("Index fold with result size ~w~n", [length(KeyList2)]), true = KeyCount div 2 == length(KeyList2), - HeadFoldFun = fun(_B, K, PO, Acc) -> {proxy_object, _MDBin, Size, _FF} = binary_to_term(PO), @@ -287,10 +286,14 @@ breaking_folds(_Config) -> end end, {async, HeadFolderToMidK} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {FoldThrowFun(HeadFoldFun), []}, - true, true, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowFun(HeadFoldFun), []}, + true, + true, + false + ), KeySizeList2 = lists:reverse(CatchingFold(HeadFolderToMidK)), io:format("Head fold with result size ~w~n", [length(KeySizeList2)]), true = KeyCount div 2 == length(KeySizeList2), @@ -300,21 +303,25 @@ breaking_folds(_Config) -> [{K,byte_size(V)}|Acc] end, {async, ObjectFolderKO} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {ObjFoldFun, []}, - false, - key_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {ObjFoldFun, []}, + false, + key_order + ), ObjSizeList1 = lists:reverse(ObjectFolderKO()), io:format("Obj fold with result size ~w~n", [length(ObjSizeList1)]), true = KeyCount == length(ObjSizeList1), {async, ObjFolderToMidK} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldThrowFun(ObjFoldFun), []}, - false, - key_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowFun(ObjFoldFun), []}, + false, + key_order + ), ObjSizeList2 = lists:reverse(CatchingFold(ObjFolderToMidK)), io:format("Object fold with result size ~w~n", [length(ObjSizeList2)]), true = KeyCount div 2 == length(ObjSizeList2), @@ -324,11 +331,13 @@ breaking_folds(_Config) -> % that was terminated by reaching a point in the key range .. as results % will not be passed to the fold function in key order {async, ObjectFolderSO} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {ObjFoldFun, []}, - false, - sqn_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {ObjFoldFun, []}, + false, + sqn_order + ), ObjSizeList1_SO = lists:reverse(ObjectFolderSO()), io:format("Obj fold with result size ~w~n", [length(ObjSizeList1_SO)]), true = KeyCount == length(ObjSizeList1_SO), @@ -346,33 +355,26 @@ breaking_folds(_Config) -> end end, {async, ObjFolderTo1K} = - leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldThrowThousandFun(ObjFoldFun), []}, - false, - sqn_order), + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + {FoldThrowThousandFun(ObjFoldFun), []}, + false, + sqn_order + ), ObjSizeList2_SO = lists:reverse(CatchingFold(ObjFolderTo1K)), io:format("Object fold with result size ~w~n", [length(ObjSizeList2_SO)]), true = 1000 == length(ObjSizeList2_SO), - ObjL2 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B2"), - ObjL3 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B3"), - ObjL4 = testutil:generate_objects(10, - binary_uuid, - [], - ObjectGen, - IndexGen, - "B4"), + ObjL2 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B2">>), + ObjL3 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B3">>), + ObjL4 = + testutil:generate_objects( + 10, binary_uuid, [], ObjectGen, IndexGen, <<"B4">>), testutil:riakload(Bookie1, ObjL2), testutil:riakload(Bookie1, ObjL3), testutil:riakload(Bookie1, ObjL4), @@ -396,20 +398,16 @@ breaking_folds(_Config) -> end, {async, StopAt3BucketFolder} = - leveled_bookie:book_bucketlist(Bookie1, - ?RIAK_TAG, - {StopAt3Fun, []}, - all), + leveled_bookie:book_bucketlist( + Bookie1, ?RIAK_TAG, {StopAt3Fun, []}, all), BucketListSA3 = lists:reverse(CatchingFold(StopAt3BucketFolder)), io:format("bucket list with result ~w~n", [BucketListSA3]), true = [<<"B2">>, <<"B3">>] == BucketListSA3, - ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). - single_object_with2i(_Config) -> % Load a single object with an integer and a binary % index and query for it @@ -429,36 +427,40 @@ single_object_with2i(_Config) -> {async, IdxFolder1} = leveled_bookie:book_indexfold( Bookie1, - "Bucket1", + <<"Bucket1">>, {fun testutil:foldkeysfun/3, []}, {list_to_binary("binary_bin"), <<99:32/integer>>, <<101:32/integer>>}, {true, undefined}), R1 = IdxFolder1(), io:format("R1 of ~w~n", [R1]), - true = [{<<100:32/integer>>,"Key1"}] == R1, + true = [{<<100:32/integer>>, <<"Key1">>}] == R1, - IdxQ2 = {index_query, - "Bucket1", - {fun testutil:foldkeysfun/3, []}, - {list_to_binary("integer_int"), - 99, 101}, - {true, undefined}}, + IdxQ2 = + { + index_query, + <<"Bucket1">>, + {fun testutil:foldkeysfun/3, []}, + {list_to_binary("integer_int"), 99, 101}, + {true, undefined} + }, {async, IdxFolder2} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2), R2 = IdxFolder2(), io:format("R2 of ~w~n", [R2]), - true = [{100,"Key1"}] == R2, + true = [{100, <<"Key1">>}] == R2, - IdxQ3 = {index_query, - {"Bucket1", "Key1"}, - {fun testutil:foldkeysfun/3, []}, - {list_to_binary("integer_int"), - 99, 101}, - {true, undefined}}, + IdxQ3 = + { + index_query, + {<<"Bucket1">>, <<"Key1">>}, + {fun testutil:foldkeysfun/3, []}, + {list_to_binary("integer_int"), 99, 101}, + {true, undefined} + }, {async, IdxFolder3} = leveled_bookie:book_returnfolder(Bookie1, IdxQ3), R3 = IdxFolder3(), io:format("R2 of ~w~n", [R3]), - true = [{100,"Key1"}] == R3, + true = [{100, <<"Key1">>}] == R3, ok = leveled_bookie:book_close(Bookie1), testutil:reset_filestructure(). @@ -473,7 +475,7 @@ small_load_with2i(_Config) -> {TestObject, TestSpec} = testutil:generate_testobject(), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), - testutil:check_formissingobject(Bookie1, "Bucket1", "Key2"), + testutil:check_formissingobject(Bookie1, <<"Bucket1">>, <<"Key2">>), testutil:check_forobject(Bookie1, TestObject), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = testutil:get_randomindexes_generator(8), @@ -486,58 +488,60 @@ small_load_with2i(_Config) -> testutil:check_forobject(Bookie1, TestObject), % Find all keys index, and then just the last key - IdxQ1 = {index_query, - "Bucket", - {fun testutil:foldkeysfun/3, []}, - {<<"idx1_bin">>, <<"#">>, <<"|">>}, - {true, undefined}}, + IdxQ1 = + { + index_query, + <<"Bucket">>, + {fun testutil:foldkeysfun/3, []}, + {<<"idx1_bin">>, <<"#">>, <<"|">>}, + {true, undefined} + }, {async, IdxFolder} = leveled_bookie:book_returnfolder(Bookie1, IdxQ1), KeyList1 = lists:usort(IdxFolder()), true = 10000 == length(KeyList1), {LastTerm, LastKey} = lists:last(KeyList1), - IdxQ2 = {index_query, - {"Bucket", LastKey}, - {fun testutil:foldkeysfun/3, []}, - {<<"idx1_bin">>, LastTerm, <<"|">>}, - {false, undefined}}, + IdxQ2 = + { + index_query, + {<<"Bucket">>, LastKey}, + {fun testutil:foldkeysfun/3, []}, + {<<"idx1_bin">>, LastTerm, <<"|">>}, + {false, undefined} + }, {async, IdxFolderLK} = leveled_bookie:book_returnfolder(Bookie1, IdxQ2), KeyList2 = lists:usort(IdxFolderLK()), io:format("List should be last key ~w ~w~n", [LastKey, KeyList2]), true = 1 == length(KeyList2), %% Delete the objects from the ChkList removing the indexes - lists:foreach(fun({_RN, Obj, Spc}) -> - DSpc = lists:map(fun({add, F, T}) -> - {remove, F, T} - end, - Spc), - {B, K} = - {testutil:get_bucket(Obj), testutil:get_key(Obj)}, - testutil:book_riakdelete(Bookie1, B, K, DSpc) - end, - ChkList1), + lists:foreach( + fun({_RN, Obj, Spc}) -> + DSpc = + lists:map(fun({add, F, T}) -> {remove, F, T} end, Spc), + {B, K} = {testutil:get_bucket(Obj), testutil:get_key(Obj)}, + testutil:book_riakdelete(Bookie1, B, K, DSpc) + end, + ChkList1 + ), %% Get the Buckets Keys and Hashes for the whole bucket - FoldObjectsFun = fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc] - end, + FoldObjectsFun = + fun(B, K, V, Acc) -> [{B, K, erlang:phash2(V)}|Acc] end, - {async, HTreeF1} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - {FoldObjectsFun, []}, - false), + {async, HTreeF1} = + leveled_bookie:book_objectfold( + Bookie1, ?RIAK_TAG, {FoldObjectsFun, []}, false), KeyHashList1 = HTreeF1(), - {async, HTreeF2} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - "Bucket", - all, - {FoldObjectsFun, []}, - false), + {async, HTreeF2} = + leveled_bookie:book_objectfold( + Bookie1, ?RIAK_TAG, <<"Bucket">>, all, {FoldObjectsFun, []}, false + ), KeyHashList2 = HTreeF2(), {async, HTreeF3} = leveled_bookie:book_objectfold( Bookie1, ?RIAK_TAG, - "Bucket", + <<"Bucket">>, {<<"idx1_bin">>, <<"#">>, <<"|">>}, {FoldObjectsFun, []}, false), @@ -546,12 +550,13 @@ small_load_with2i(_Config) -> true = 9900 == length(KeyHashList2), true = 9900 == length(KeyHashList3), - SumIntFun = fun(_B, _K, Obj, Acc) -> - {I, _Bin} = testutil:get_value(Obj), - Acc + I - end, + SumIntFun = + fun(_B, _K, Obj, Acc) -> + {I, _Bin} = testutil:get_value(Obj), + Acc + I + end, BucketObjQ = - {foldobjects_bybucket, ?RIAK_TAG, "Bucket", all, {SumIntFun, 0}, true}, + {foldobjects_bybucket, ?RIAK_TAG, <<"Bucket">>, all, {SumIntFun, 0}, true}, {async, Sum1} = leveled_bookie:book_returnfolder(Bookie1, BucketObjQ), Total1 = Sum1(), io:format("Total from summing all I is ~w~n", [Total1]), @@ -596,21 +601,18 @@ query_count(_Config) -> BucketBin = list_to_binary("Bucket"), {TestObject, TestSpec} = testutil:generate_testobject( - BucketBin, term_to_binary("Key1"), "Value1", [], [{"MDK1", "MDV1"}]), + BucketBin, term_to_binary("Key1"), <<"Value1">>, [], [{<<"MDK1">>, <<"MDV1">>}]), ok = testutil:book_riakput(Book1, TestObject, TestSpec), testutil:check_forobject(Book1, TestObject), - testutil:check_formissingobject(Book1, "Bucket1", "Key2"), + testutil:check_formissingobject(Book1, <<"Bucket1">>, <<"Key2">>), testutil:check_forobject(Book1, TestObject), lists:foreach( fun(_X) -> V = testutil:get_compressiblevalue(), Indexes = testutil:get_randomindexes_generator(8), SW = os:timestamp(), - ObjL1 = testutil:generate_objects(10000, - binary_uuid, - [], - V, - Indexes), + ObjL1 = + testutil:generate_objects(10000, binary_uuid, [], V, Indexes), testutil:riakload(Book1, ObjL1), io:format( "Put of 10000 objects with 8 index entries " @@ -681,15 +683,17 @@ query_count(_Config) -> {true, undefined}}, {async, Mia2KFolder2} = leveled_bookie:book_returnfolder(Book2, Query2), - Mia2000Count2 = lists:foldl(fun({Term, _Key}, Acc) -> - case re:run(Term, RegMia) of - nomatch -> - Acc; - _ -> - Acc + 1 - end end, - 0, - Mia2KFolder2()), + Mia2000Count2 = + lists:foldl( + fun({Term, _Key}, Acc) -> + case re:run(Term, RegMia) of + nomatch -> + Acc; + _ -> + Acc + 1 + end end, + 0, + Mia2KFolder2()), ok = case Mia2000Count2 of Mia2000Count1 when Mia2000Count1 > 0 -> io:format("Mia2000 counts match at ~w~n", @@ -731,20 +735,22 @@ query_count(_Config) -> Spc9Del = lists:map(fun({add, IdxF, IdxT}) -> {remove, IdxF, IdxT} end, Spc9), ok = testutil:book_riakput(Book2, Obj9, Spc9Del), - lists:foreach(fun({IdxF, IdxT, X}) -> - Q = {index_query, - BucketBin, - {fun testutil:foldkeysfun/3, []}, - {IdxF, IdxT, IdxT}, - ?KEY_ONLY}, - R = leveled_bookie:book_returnfolder(Book2, Q), - {async, Fldr} = R, - case length(Fldr()) of - Y -> - Y = X - 1 - end - end, - R9), + lists:foreach( + fun({IdxF, IdxT, X}) -> + Q = {index_query, + BucketBin, + {fun testutil:foldkeysfun/3, []}, + {IdxF, IdxT, IdxT}, + ?KEY_ONLY}, + R = leveled_bookie:book_returnfolder(Book2, Q), + {async, Fldr} = R, + case length(Fldr()) of + Y -> + Y = X - 1 + end + end, + R9 + ), ok = leveled_bookie:book_close(Book2), {ok, Book3} = leveled_bookie:book_start( @@ -800,13 +806,13 @@ query_count(_Config) -> ObjList10A = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketA"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketA">>), ObjList10B = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketB"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketB">>), ObjList10C = testutil:generate_objects( - 5000, binary_uuid, [], V9, Indexes9, "BucketC"), + 5000, binary_uuid, [], V9, Indexes9, <<"BucketC">>), testutil:riakload(Book4, ObjList10A), testutil:riakload(Book4, ObjList10B), testutil:riakload(Book4, ObjList10C), @@ -819,10 +825,9 @@ query_count(_Config) -> ok = leveled_bookie:book_close(Book4), - {ok, Book5} = leveled_bookie:book_start(RootPath, - 2000, - 50000000, - testutil:sync_strategy()), + {ok, Book5} = + leveled_bookie:book_start( + RootPath, 2000, 50000000, testutil:sync_strategy()), {async, BLF3} = leveled_bookie:book_returnfolder(Book5, BucketListQuery), SW_QC = os:timestamp(), BucketSet3 = BLF3(), @@ -866,33 +871,25 @@ multibucket_fold(_Config) -> testutil:sync_strategy()), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = fun() -> [] end, - ObjL1 = testutil:generate_objects(13000, - uuid, - [], - ObjectGen, - IndexGen, - {<<"Type1">>, <<"Bucket1">>}), + ObjL1 = + testutil:generate_objects( + 13000, uuid, [], ObjectGen, IndexGen, {<<"Type1">>, <<"Bucket1">>} + ), testutil:riakload(Bookie1, ObjL1), - ObjL2 = testutil:generate_objects(17000, - uuid, - [], - ObjectGen, - IndexGen, - <<"Bucket2">>), + ObjL2 = + testutil:generate_objects( + 17000, uuid, [], ObjectGen, IndexGen, <<"Bucket2">> + ), testutil:riakload(Bookie1, ObjL2), - ObjL3 = testutil:generate_objects(7000, - uuid, - [], - ObjectGen, - IndexGen, - <<"Bucket3">>), + ObjL3 = + testutil:generate_objects( + 7000, uuid, [], ObjectGen, IndexGen, <<"Bucket3">> + ), testutil:riakload(Bookie1, ObjL3), - ObjL4 = testutil:generate_objects(23000, - uuid, - [], - ObjectGen, - IndexGen, - {<<"Type2">>, <<"Bucket4">>}), + ObjL4 = + testutil:generate_objects( + 23000, uuid, [], ObjectGen, IndexGen, {<<"Type2">>, <<"Bucket4">>} + ), testutil:riakload(Bookie1, ObjL4), FF = fun(B, K, _PO, Acc) -> @@ -901,30 +898,30 @@ multibucket_fold(_Config) -> FoldAccT = {FF, []}, {async, R1} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, - [{<<"Type1">>, <<"Bucket1">>}, - {<<"Type2">>, <<"Bucket4">>}]}, - FoldAccT, - false, - true, - false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, + [{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>}]}, + FoldAccT, + false, + true, + false + ), O1 = length(R1()), io:format("Result R1 of length ~w~n", [O1]), {async, R2} = - leveled_bookie:book_headfold(Bookie1, - ?RIAK_TAG, - {bucket_list, - [<<"Bucket2">>, - <<"Bucket3">>]}, - {fun(_B, _K, _PO, Acc) -> - Acc +1 - end, - 0}, - false, true, false), + leveled_bookie:book_headfold( + Bookie1, + ?RIAK_TAG, + {bucket_list, [<<"Bucket2">>, <<"Bucket3">>]}, + {fun(_B, _K, _PO, Acc) -> Acc +1 end, 0}, + false, + true, + false + ), O2 = R2(), io:format("Result R2 of ~w~n", [O2]), @@ -933,10 +930,8 @@ multibucket_fold(_Config) -> FoldBucketsFun = fun(B, Acc) -> [B|Acc] end, {async, Folder} = - leveled_bookie:book_bucketlist(Bookie1, - ?RIAK_TAG, - {FoldBucketsFun, []}, - all), + leveled_bookie:book_bucketlist( + Bookie1, ?RIAK_TAG, {FoldBucketsFun, []}, all), BucketList = lists:reverse(Folder()), ExpectedBucketList = [{<<"Type1">>, <<"Bucket1">>}, {<<"Type2">>, <<"Bucket4">>}, @@ -949,54 +944,53 @@ multibucket_fold(_Config) -> rotating_objects(_Config) -> RootPath = testutil:reset_filestructure(), - ok = testutil:rotating_object_check(RootPath, "Bucket1", 10), - ok = testutil:rotating_object_check(RootPath, "Bucket2", 200), - ok = testutil:rotating_object_check(RootPath, "Bucket3", 800), - ok = testutil:rotating_object_check(RootPath, "Bucket4", 1600), - ok = testutil:rotating_object_check(RootPath, "Bucket5", 3200), - ok = testutil:rotating_object_check(RootPath, "Bucket6", 9600), + ok = testutil:rotating_object_check(RootPath, <<"Bucket1">>, 10), + ok = testutil:rotating_object_check(RootPath, <<"Bucket2">>, 200), + ok = testutil:rotating_object_check(RootPath, <<"Bucket3">>, 800), + ok = testutil:rotating_object_check(RootPath, <<"Bucket4">>, 1600), + ok = testutil:rotating_object_check(RootPath, <<"Bucket5">>, 3200), + ok = testutil:rotating_object_check(RootPath, <<"Bucket6">>, 9600), testutil:reset_filestructure(). foldobjects_bybucket_range(_Config) -> RootPath = testutil:reset_filestructure(), - {ok, Bookie1} = leveled_bookie:book_start(RootPath, - 2000, - 50000000, - testutil:sync_strategy()), + {ok, Bookie1} = + leveled_bookie:book_start( + RootPath, 2000, 50000000, testutil:sync_strategy()), ObjectGen = testutil:get_compressiblevalue_andinteger(), IndexGen = fun() -> [] end, - ObjL1 = testutil:generate_objects(1300, - {fixed_binary, 1}, - [], - ObjectGen, - IndexGen, - <<"Bucket1">>), + ObjL1 = + testutil:generate_objects( + 1300, {fixed_binary, 1}, [], ObjectGen, IndexGen, <<"Bucket1">>), testutil:riakload(Bookie1, ObjL1), - FoldKeysFun = fun(_B, K,_V, Acc) -> - [ K |Acc] - end, + FoldKeysFun = fun(_B, K,_V, Acc) -> [ K |Acc] end, StartKey = testutil:fixed_bin_key(123), EndKey = testutil:fixed_bin_key(779), - {async, Folder} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - <<"Bucket1">>, - {StartKey, EndKey}, {FoldKeysFun, []}, - true - ), + {async, Folder} = + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + {StartKey, EndKey}, + {FoldKeysFun, []}, + true + ), ResLen = length(Folder()), io:format("Length of Result of folder ~w~n", [ResLen]), true = 657 == ResLen, - {async, AllFolder} = leveled_bookie:book_objectfold(Bookie1, - ?RIAK_TAG, - <<"Bucket1">>, - all, - {FoldKeysFun, []}, - true - ), + {async, AllFolder} = + leveled_bookie:book_objectfold( + Bookie1, + ?RIAK_TAG, + <<"Bucket1">>, + all, + {FoldKeysFun, []}, + true + ), AllResLen = length(AllFolder()), io:format("Length of Result of all keys folder ~w~n", [AllResLen]), diff --git a/test/end_to_end/perf_SUITE.erl b/test/end_to_end/perf_SUITE.erl index c4d40f38..59d746fd 100644 --- a/test/end_to_end/perf_SUITE.erl +++ b/test/end_to_end/perf_SUITE.erl @@ -101,7 +101,7 @@ riak_load_tester(Bucket, KeyCount, ObjSize, ProfileList, PM, LC) -> IndexGenFun = fun(ListID) -> fun() -> - RandInt = leveled_rand:uniform(IndexCount - 1), + RandInt = rand:uniform(IndexCount - 1), IntIndex = ["integer", integer_to_list(ListID), "_int"], BinIndex = ["binary", integer_to_list(ListID), "_bin"], [{add, iolist_to_binary(IntIndex), RandInt}, @@ -434,35 +434,35 @@ rotation_withnocheck(Book, B, NumberOfObjects, ObjSize, IdxCnt) -> Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), rotation_with_prefetch( Book, B, NumberOfObjects, - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IdxCnt ), ok. @@ -471,7 +471,7 @@ generate_chunk(CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) -> testutil:generate_objects( CountPerList, {fixed_binary, (Chunk - 1) * CountPerList + 1}, [], - base64:encode(leveled_rand:rand_bytes(ObjSize)), + base64:encode(crypto:strong_rand_bytes(ObjSize)), IndexGenFun(Chunk), Bucket ). @@ -480,7 +480,7 @@ load_chunk(Bookie, CountPerList, ObjSize, IndexGenFun, Bucket, Chunk) -> ct:log(?INFO, "Generating and loading ObjList ~w", [Chunk]), time_load_chunk( Bookie, - {Bucket, base64:encode(leveled_rand:rand_bytes(ObjSize)), IndexGenFun(Chunk)}, + {Bucket, base64:encode(crypto:strong_rand_bytes(ObjSize)), IndexGenFun(Chunk)}, (Chunk - 1) * CountPerList + 1, Chunk * CountPerList, 0, @@ -577,9 +577,9 @@ random_fetches(FetchType, Bookie, Bucket, ObjCount, Fetches) -> case I rem 5 of 1 -> testutil:fixed_bin_key( - Twenty + leveled_rand:uniform(ObjCount - Twenty)); + Twenty + rand:uniform(ObjCount - Twenty)); _ -> - testutil:fixed_bin_key(leveled_rand:uniform(Twenty)) + testutil:fixed_bin_key(rand:uniform(Twenty)) end end, {TC, ok} = @@ -616,18 +616,18 @@ random_fetches(FetchType, Bookie, Bucket, ObjCount, Fetches) -> random_queries(Bookie, Bucket, IDs, IdxCnt, MaxRange, IndexesReturned) -> QueryFun = fun() -> - ID = leveled_rand:uniform(IDs), + ID = rand:uniform(IDs), BinIndex = iolist_to_binary(["binary", integer_to_list(ID), "_bin"]), Twenty = IdxCnt div 5, - RI = leveled_rand:uniform(MaxRange), + RI = rand:uniform(MaxRange), [Start, End] = case RI of RI when RI < (MaxRange div 5) -> - R0 = leveled_rand:uniform(IdxCnt - (Twenty + RI)), + R0 = rand:uniform(IdxCnt - (Twenty + RI)), [R0 + Twenty, R0 + Twenty + RI]; _ -> - R0 = leveled_rand:uniform(Twenty - RI), + R0 = rand:uniform(Twenty - RI), [R0, R0 + RI] end, FoldKeysFun = fun(_B, _K, Cnt) -> Cnt + 1 end, diff --git a/test/end_to_end/riak_SUITE.erl b/test/end_to_end/riak_SUITE.erl index b2c84d11..e5288c79 100644 --- a/test/end_to_end/riak_SUITE.erl +++ b/test/end_to_end/riak_SUITE.erl @@ -58,7 +58,7 @@ basic_riak_tester(Bucket, KeyCount) -> IndexGenFun = fun(ListID) -> fun() -> - RandInt = leveled_rand:uniform(IndexCount), + RandInt = rand:uniform(IndexCount), ID = integer_to_list(ListID), [{add, list_to_binary("integer" ++ ID ++ "_int"), @@ -75,7 +75,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(1), Bucket ), @@ -83,7 +83,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(2), Bucket ), @@ -92,7 +92,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 2 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(3), Bucket ), @@ -101,7 +101,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 3 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(4), Bucket ), @@ -110,7 +110,7 @@ basic_riak_tester(Bucket, KeyCount) -> testutil:generate_objects( CountPerList, {fixed_binary, 4 * CountPerList + 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), IndexGenFun(5), Bucket ), @@ -276,7 +276,7 @@ summarisable_sstindex(_Config) -> ObjListToSort = lists:map( fun(I) -> - {leveled_rand:uniform(KeyCount * 10), + {rand:uniform(KeyCount * 10), testutil:set_object( Bucket, KeyGen(I), integer_to_binary(I), IndexGen, [])} end, @@ -344,7 +344,7 @@ summarisable_sstindex(_Config) -> true = 200 == length(KeyRangeCheckFun(StartKey, EndKey)) end, lists:map( - fun(_I) -> leveled_rand:uniform(KeyCount - 200) end, + fun(_I) -> rand:uniform(KeyCount - 200) end, lists:seq(1, 100))), IdxObjKeyCount = 50000, @@ -367,7 +367,7 @@ summarisable_sstindex(_Config) -> IdxObjListToSort = lists:map( fun(I) -> - {leveled_rand:uniform(KeyCount * 10), + {rand:uniform(KeyCount * 10), testutil:set_object( Bucket, KeyGen(I), @@ -419,7 +419,7 @@ summarisable_sstindex(_Config) -> end, lists:map( fun(_I) -> - leveled_rand:uniform(IdxObjKeyCount - 20) + rand:uniform(IdxObjKeyCount - 20) end, lists:seq(1, 100))), lists:foreach( @@ -430,7 +430,7 @@ summarisable_sstindex(_Config) -> end, lists:map( fun(_I) -> - leveled_rand:uniform(IdxObjKeyCount - 10) + rand:uniform(IdxObjKeyCount - 10) end, lists:seq(1, 100))), @@ -451,7 +451,7 @@ summarisable_sstindex(_Config) -> true = 200 == length(KeyRangeCheckFun(StartKey, EndKey)) end, lists:map( - fun(_I) -> leveled_rand:uniform(KeyCount - 200) end, + fun(_I) -> rand:uniform(KeyCount - 200) end, lists:seq(1, 100))), ok = leveled_bookie:book_destroy(Bookie1). @@ -475,7 +475,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 100000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(32), + crypto:strong_rand_bytes(32), fun() -> [] end, <<"BaselineB">> ), @@ -485,7 +485,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 20000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -498,7 +498,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 15000, {fixed_binary, 20001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -511,7 +511,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 35000, {fixed_binary, 35001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -524,7 +524,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 30000, {fixed_binary, 70001}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B0">> ), @@ -537,7 +537,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 8000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B1">> ), @@ -550,7 +550,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 7000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(512), + crypto:strong_rand_bytes(512), fun() -> [] end, <<"B2">> ), @@ -815,7 +815,7 @@ fetchclocks_modifiedbetween(_Config) -> testutil:generate_objects( 200000, {fixed_binary, 1}, [], - leveled_rand:rand_bytes(32), + crypto:strong_rand_bytes(32), fun() -> [] end, <<"B1.9">> ), @@ -1637,7 +1637,7 @@ bigobject_memorycheck(_Config) -> ObjPutFun = fun(I) -> Key = base64:encode(<>), - Value = leveled_rand:rand_bytes(1024 * 1024), + Value = crypto:strong_rand_bytes(1024 * 1024), % a big object each time! {Obj, Spc} = testutil:set_object(Bucket, Key, Value, IndexGen, []), testutil:book_riakput(Bookie, Obj, Spc) diff --git a/test/end_to_end/testutil.erl b/test/end_to_end/testutil.erl index 82c02350..5816d112 100644 --- a/test/end_to_end/testutil.erl +++ b/test/end_to_end/testutil.erl @@ -231,12 +231,14 @@ sync_strategy() -> none. book_riakput(Pid, RiakObject, IndexSpecs) -> - leveled_bookie:book_put(Pid, - RiakObject#r_object.bucket, - RiakObject#r_object.key, - to_binary(v1, RiakObject), - IndexSpecs, - ?RIAK_TAG). + leveled_bookie:book_put( + Pid, + RiakObject#r_object.bucket, + RiakObject#r_object.key, + to_binary(v1, RiakObject), + IndexSpecs, + ?RIAK_TAG + ). book_tempriakput(Pid, RiakObject, IndexSpecs, TTL) -> leveled_bookie:book_tempput( @@ -246,7 +248,8 @@ book_tempriakput(Pid, RiakObject, IndexSpecs, TTL) -> to_binary(v1, RiakObject), IndexSpecs, ?RIAK_TAG, - TTL). + TTL + ). book_riakdelete(Pid, Bucket, Key, IndexSpecs) -> leveled_bookie:book_put(Pid, Bucket, Key, delete, IndexSpecs, ?RIAK_TAG). @@ -383,9 +386,8 @@ wait_for_compaction(Bookie) -> check_bucket_stats(Bookie, Bucket) -> FoldSW1 = os:timestamp(), io:format("Checking bucket size~n"), - {async, Folder1} = leveled_bookie:book_returnfolder(Bookie, - {riakbucket_stats, - Bucket}), + {async, Folder1} = + leveled_bookie:book_returnfolder(Bookie, {riakbucket_stats, Bucket}), {B1Size, B1Count} = Folder1(), io:format("Bucket fold completed in ~w microseconds~n", [timer:now_diff(os:timestamp(), FoldSW1)]), @@ -399,28 +401,32 @@ check_forlist(Bookie, ChkList) -> check_forlist(Bookie, ChkList, Log) -> SW = os:timestamp(), - lists:foreach(fun({_RN, Obj, _Spc}) -> - if - Log == true -> - io:format("Fetching Key ~s~n", [Obj#r_object.key]); - true -> - ok - end, - R = book_riakget(Bookie, - Obj#r_object.bucket, - Obj#r_object.key), - true = case R of - {ok, Val} -> - to_binary(v1, Obj) == Val; - not_found -> - io:format("Object not found for key ~s~n", - [Obj#r_object.key]), - error - end - end, - ChkList), - io:format("Fetch check took ~w microseconds checking list of length ~w~n", - [timer:now_diff(os:timestamp(), SW), length(ChkList)]). + lists:foreach( + fun({_RN, Obj, _Spc}) -> + if + Log == true -> + io:format("Fetching Key ~s~n", [Obj#r_object.key]); + true -> + ok + end, + R = book_riakget(Bookie, + Obj#r_object.bucket, + Obj#r_object.key), + true = + case R of + {ok, Val} -> + to_binary(v1, Obj) == Val; + not_found -> + io:format("Object not found for key ~s~n", + [Obj#r_object.key]), + error + end + end, + ChkList), + io:format( + "Fetch check took ~w microseconds checking list of length ~w~n", + [timer:now_diff(os:timestamp(), SW), length(ChkList)] + ). checkhead_forlist(Bookie, ChkList) -> SW = os:timestamp(), @@ -470,11 +476,14 @@ check_formissingobject(Bookie, Bucket, Key) -> generate_testobject() -> - {B1, K1, V1, Spec1, MD} = {"Bucket1", - "Key1", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, Spec1, MD} = + { + <<"Bucket1">>, + <<"Key1">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, generate_testobject(B1, K1, V1, Spec1, MD). generate_testobject(B, K, V, Spec, MD) -> @@ -493,7 +502,7 @@ generate_compressibleobjects(Count, KeyNumber) -> get_compressiblevalue_andinteger() -> - {leveled_rand:uniform(1000), get_compressiblevalue()}. + {rand:uniform(1000), get_compressiblevalue()}. get_compressiblevalue() -> S1 = "111111111111111", @@ -510,7 +519,7 @@ get_compressiblevalue() -> iolist_to_binary( lists:foldl( fun(_X, Acc) -> - {_, Str} = lists:keyfind(leveled_rand:uniform(8), 1, Selector), + {_, Str} = lists:keyfind(rand:uniform(8), 1, Selector), [Str|Acc] end, [""], L @@ -518,28 +527,39 @@ get_compressiblevalue() -> ). generate_smallobjects(Count, KeyNumber) -> - generate_objects(Count, KeyNumber, [], leveled_rand:rand_bytes(512)). + generate_objects(Count, KeyNumber, [], crypto:strong_rand_bytes(512)). generate_objects(Count, KeyNumber) -> - generate_objects(Count, KeyNumber, [], leveled_rand:rand_bytes(4096)). + generate_objects(Count, KeyNumber, [], crypto:strong_rand_bytes(4096)). generate_objects(Count, KeyNumber, ObjL, Value) -> generate_objects(Count, KeyNumber, ObjL, Value, fun() -> [] end). generate_objects(Count, KeyNumber, ObjL, Value, IndexGen) -> - generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, "Bucket"). + generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, <<"Bucket">>). generate_objects(0, _KeyNumber, ObjL, _Value, _IndexGen, _Bucket) -> lists:reverse(ObjL); -generate_objects(Count, binary_uuid, ObjL, Value, IndexGen, Bucket) -> - {Obj1, Spec1} = set_object(list_to_binary(Bucket), - list_to_binary(leveled_util:generate_uuid()), - Value, - IndexGen), +generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, Bucket) + when is_list(Bucket) -> + generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, list_to_binary(Bucket) + ); +generate_objects( + Count, binary_uuid, ObjL, Value, IndexGen, Bucket) + when is_binary(Bucket) -> + {Obj1, Spec1} = + set_object( + Bucket, + list_to_binary(leveled_util:generate_uuid()), + Value, + IndexGen + ), generate_objects(Count - 1, binary_uuid, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -550,19 +570,29 @@ generate_objects(Count, uuid, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, uuid, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); -generate_objects(Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) -> +generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) + when is_list(Bucket) -> + generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, list_to_binary(Bucket) + ); +generate_objects( + Count, {binary, KeyNumber}, ObjL, Value, IndexGen, Bucket) + when is_binary(Bucket) -> {Obj1, Spec1} = - set_object(list_to_binary(Bucket), - list_to_binary(numbered_key(KeyNumber)), - Value, - IndexGen), + set_object( + Bucket, + list_to_binary(numbered_key(KeyNumber)), + Value, + IndexGen + ), generate_objects(Count - 1, {binary, KeyNumber + 1}, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -574,7 +604,7 @@ generate_objects(Count, {fixed_binary, KeyNumber}, ObjL, Value, IndexGen, Bucket IndexGen), generate_objects(Count - 1, {fixed_binary, KeyNumber + 1}, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket); @@ -585,7 +615,7 @@ generate_objects(Count, KeyNumber, ObjL, Value, IndexGen, Bucket) -> IndexGen), generate_objects(Count - 1, KeyNumber + 1, - [{leveled_rand:uniform(), Obj1, Spec1}|ObjL], + [{rand:uniform(), Obj1, Spec1}|ObjL], Value, IndexGen, Bucket). @@ -652,7 +682,7 @@ update_some_objects(Bookie, ObjList, SampleSize) -> [C] = Obj#r_object.contents, MD = C#r_content.metadata, MD0 = dict:store(?MD_LASTMOD, os:timestamp(), MD), - C0 = C#r_content{value = leveled_rand:rand_bytes(512), + C0 = C#r_content{value = crypto:strong_rand_bytes(512), metadata = MD0}, UpdObj = Obj#r_object{vclock = VC0, contents = [C0]}, {R, UpdObj, Spec} @@ -679,11 +709,11 @@ delete_some_objects(Bookie, ObjList, SampleSize) -> generate_vclock() -> lists:map(fun(X) -> - {_, Actor} = lists:keyfind(leveled_rand:uniform(10), + {_, Actor} = lists:keyfind(rand:uniform(10), 1, actor_list()), {Actor, X} end, - lists:seq(1, leveled_rand:uniform(8))). + lists:seq(1, rand:uniform(8))). update_vclock(VC) -> [{Actor, X}|Rest] = VC, @@ -785,14 +815,14 @@ name_list() -> get_randomname() -> NameList = name_list(), - N = leveled_rand:uniform(16), + N = rand:uniform(16), {N, Name} = lists:keyfind(N, 1, NameList), Name. get_randomdate() -> LowTime = 60000000000, HighTime = 70000000000, - RandPoint = LowTime + leveled_rand:uniform(HighTime - LowTime), + RandPoint = LowTime + rand:uniform(HighTime - LowTime), Date = calendar:gregorian_seconds_to_datetime(RandPoint), {{Year, Month, Day}, {Hour, Minute, Second}} = Date, lists:flatten(io_lib:format("~4..0w~2..0w~2..0w~2..0w~2..0w~2..0w", diff --git a/test/end_to_end/tictac_SUITE.erl b/test/end_to_end/tictac_SUITE.erl index 277a7a3d..69b4f8d9 100644 --- a/test/end_to_end/tictac_SUITE.erl +++ b/test/end_to_end/tictac_SUITE.erl @@ -41,11 +41,14 @@ many_put_compare(_Config) -> {max_pencillercachesize, 16000}, {sync_strategy, riak_sync}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {B1, K1, V1, S1, MD} = {"Bucket", - "Key1.1.4567.4321", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, S1, MD} = + { + <<"Bucket">>, + <<"Key1.1.4567.4321">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), ok = testutil:book_riakput(Bookie1, TestObject, TestSpec), testutil:check_forobject(Bookie1, TestObject), @@ -63,12 +66,15 @@ many_put_compare(_Config) -> GenList = [2, 20002, 40002, 60002, 80002, 100002, 120002, 140002, 160002, 180002], - CLs = testutil:load_objects(20000, - GenList, - Bookie2, - TestObject, - fun testutil:generate_smallobjects/2, - 20000), + CLs = + testutil:load_objects( + 20000, + GenList, + Bookie2, + TestObject, + fun testutil:generate_smallobjects/2, + 20000 + ), % Start a new store, and load the same objects (except fot the original % test object) into this store @@ -84,7 +90,7 @@ many_put_compare(_Config) -> % state between stores is consistent TicTacQ = {tictactree_obj, - {o_rkv, "Bucket", null, null, true}, + {o_rkv, <<"Bucket">>, null, null, true}, TreeSize, fun(_B, _K) -> accumulate end}, {async, TreeAFolder} = leveled_bookie:book_returnfolder(Bookie2, TicTacQ), @@ -113,10 +119,13 @@ many_put_compare(_Config) -> true = length(AltList) > 10000, % check there are a significant number of differences from empty - WrongPartitionTicTacQ = {tictactree_obj, - {o_rkv, "Bucket", null, null, false}, - TreeSize, - fun(_B, _K) -> pass end}, + WrongPartitionTicTacQ = + { + tictactree_obj, + {o_rkv, <<"Bucket">>, null, null, false}, + TreeSize, + fun(_B, _K) -> pass end + }, {async, TreeAFolder_WP} = leveled_bookie:book_returnfolder(Bookie2, WrongPartitionTicTacQ), TreeAWP = TreeAFolder_WP(), @@ -151,7 +160,7 @@ many_put_compare(_Config) -> {async, TreeAObjFolder0} = leveled_bookie:book_headfold(Bookie2, o_rkv, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, FoldAccT, false, true, @@ -170,7 +179,7 @@ many_put_compare(_Config) -> leveled_bookie:book_headfold( Bookie2, ?RIAK_TAG, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, {FoldObjectsFun, InitAccTree}, true, true, @@ -188,7 +197,7 @@ many_put_compare(_Config) -> leveled_bookie:book_headfold( Bookie2, ?RIAK_TAG, - {range, "Bucket", all}, + {range, <<"Bucket">>, all}, {FoldObjectsFun, leveled_tictac:new_tree(0, TreeSize, false)}, true, true, @@ -218,29 +227,38 @@ many_put_compare(_Config) -> end, {async, TreeAAltObjFolder0} = - leveled_bookie:book_headfold(Bookie2, - ?RIAK_TAG, - {range, "Bucket", all}, - {AltFoldObjectsFun, - InitAccTree}, - false, true, false), + leveled_bookie:book_headfold( + Bookie2, + ?RIAK_TAG, + {range, <<"Bucket">>, all}, + {AltFoldObjectsFun, InitAccTree}, + false, + true, + false + ), SWB2Obj = os:timestamp(), TreeAAltObj = TreeAAltObjFolder0(), - io:format("Build tictac tree via object fold with no "++ - "presence check and 200K objects and alt hash in ~w~n", - [timer:now_diff(os:timestamp(), SWB2Obj)]), + io:format( + "Build tictac tree via object fold with no " + "presence check and 200K objects and alt hash in ~w~n", + [timer:now_diff(os:timestamp(), SWB2Obj)] + ), {async, TreeBAltObjFolder0} = - leveled_bookie:book_headfold(Bookie3, - ?RIAK_TAG, - {range, "Bucket", all}, - {AltFoldObjectsFun, - InitAccTree}, - false, true, false), + leveled_bookie:book_headfold( + Bookie3, + ?RIAK_TAG, + {range, <<"Bucket">>, all}, + {AltFoldObjectsFun, InitAccTree}, + false, + true, + false + ), SWB3Obj = os:timestamp(), TreeBAltObj = TreeBAltObjFolder0(), - io:format("Build tictac tree via object fold with no "++ - "presence check and 200K objects and alt hash in ~w~n", - [timer:now_diff(os:timestamp(), SWB3Obj)]), + io:format( + "Build tictac tree via object fold with no " + "presence check and 200K objects and alt hash in ~w~n", + [timer:now_diff(os:timestamp(), SWB3Obj)]), DL_ExportFold = length(leveled_tictac:find_dirtyleaves(TreeBAltObj, TreeAAltObj)), io:format("Found dirty leaves with exportable comparison of ~w~n", @@ -261,7 +279,7 @@ many_put_compare(_Config) -> end end end, - SegQuery = {keylist, o_rkv, "Bucket", {FoldKeysFun(SegList0), []}}, + SegQuery = {keylist, o_rkv, <<"Bucket">>, {FoldKeysFun(SegList0), []}}, {async, SegKeyFinder} = leveled_bookie:book_returnfolder(Bookie2, SegQuery), SWSKL0 = os:timestamp(), @@ -273,7 +291,7 @@ many_put_compare(_Config) -> true = length(SegKeyList) >= 1, true = length(SegKeyList) < 10, - true = lists:member("Key1.1.4567.4321", SegKeyList), + true = lists:member(<<"Key1.1.4567.4321">>, SegKeyList), % Now remove the object which represents the difference between these % stores and confirm that the tictac trees will now match @@ -630,20 +648,23 @@ tuplebuckets_headonly(_Config) -> SW1 = os:timestamp(), {async, HeadRunner1} = - leveled_bookie:book_headfold(Bookie1, - ?HEAD_TAG, - {bucket_list, BucketList}, - {FoldHeadFun, []}, - false, false, - false), + leveled_bookie:book_headfold( + Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + false + ), ReturnedObjSpecL1 = lists:reverse(HeadRunner1()), [FirstItem|_Rest] = ReturnedObjSpecL1, LastItem = lists:last(ReturnedObjSpecL1), - io:format("Returned ~w objects with first ~w and last ~w in ~w ms~n", - [length(ReturnedObjSpecL1), - FirstItem, LastItem, - timer:now_diff(os:timestamp(), SW1)/1000]), + io:format( + "Returned ~w objects with first ~w and last ~w in ~w ms~n", + [length(ReturnedObjSpecL1), + FirstItem, LastItem, + timer:now_diff(os:timestamp(), SW1)/1000]), true = ReturnedObjSpecL1 == lists:sort(ObjectSpecL), @@ -654,12 +675,14 @@ tuplebuckets_headonly(_Config) -> SW2 = os:timestamp(), {async, HeadRunner2} = - leveled_bookie:book_headfold(Bookie1, - ?HEAD_TAG, - {bucket_list, BucketList}, - {FoldHeadFun, []}, - false, false, - SegList), + leveled_bookie:book_headfold( + Bookie1, + ?HEAD_TAG, + {bucket_list, BucketList}, + {FoldHeadFun, []}, + false, false, + SegList + ), ReturnedObjSpecL2 = lists:reverse(HeadRunner2()), io:format("Returned ~w objects using seglist in ~w ms~n", @@ -674,7 +697,6 @@ tuplebuckets_headonly(_Config) -> leveled_bookie:book_destroy(Bookie1). - basic_headonly(_Config) -> ObjectCount = 200000, RemoveCount = 100, @@ -694,11 +716,14 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> {head_only, HeadOnly}, {max_journalsize, 500000}], {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), - {B1, K1, V1, S1, MD} = {"Bucket", - "Key1.1.4567.4321", - "Value1", - [], - [{"MDK1", "MDV1"}]}, + {B1, K1, V1, S1, MD} = + { + <<"Bucket">>, + <<"Key1.1.4567.4321">>, + <<"Value1">>, + [], + [{<<"MDK1">>, <<"MDV1">>}] + }, {TestObject, TestSpec} = testutil:generate_testobject(B1, K1, V1, S1, MD), {unsupported_message, put} = testutil:book_riakput(Bookie1, TestObject, TestSpec), @@ -818,23 +843,21 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> false = is_process_alive(AltSnapshot); no_lookup -> {unsupported_message, head} = - leveled_bookie:book_head(Bookie1, - SegmentID0, - {Bucket0, Key0}, - h), + leveled_bookie:book_head( + Bookie1, SegmentID0, {Bucket0, Key0}, h), {unsupported_message, head} = - leveled_bookie:book_headonly(Bookie1, - SegmentID0, - Bucket0, - Key0), + leveled_bookie:book_headonly( + Bookie1, SegmentID0, Bucket0, Key0), io:format("Closing actual store ~w~n", [Bookie1]), ok = leveled_bookie:book_close(Bookie1) end, {ok, FinalJournals} = file:list_dir(JFP), - io:format("Trim has reduced journal count from " ++ - "~w to ~w and ~w after restart~n", - [length(FNs), length(FinalFNs), length(FinalJournals)]), + io:format( + "Trim has reduced journal count from " + "~w to ~w and ~w after restart~n", + [length(FNs), length(FinalFNs), length(FinalJournals)] + ), {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), @@ -849,16 +872,12 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> % If we allow HEAD_TAG to be suubject to a lookup, then test this % here {ok, Hash0} = - leveled_bookie:book_head(Bookie2, - SegmentID0, - {Bucket0, Key0}, - h); + leveled_bookie:book_head( + Bookie2, SegmentID0, {Bucket0, Key0}, h); no_lookup -> {unsupported_message, head} = - leveled_bookie:book_head(Bookie2, - SegmentID0, - {Bucket0, Key0}, - h) + leveled_bookie:book_head( + Bookie2, SegmentID0, {Bucket0, Key0}, h) end, RemoveSpecL0 = lists:sublist(ObjectSpecL, RemoveCount), @@ -873,12 +892,9 @@ basic_headonly_test(ObjectCount, RemoveCount, HeadOnly) -> true = AccC3 == (ObjectCount - RemoveCount), false = AccH3 == AccH2, - ok = leveled_bookie:book_close(Bookie2). - - load_objectspecs([], _SliceSize, _Bookie) -> ok; load_objectspecs(ObjectSpecL, SliceSize, Bookie)