From 9157de680ed5e46e496a48883bf02aea0adbc555 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 11 Jan 2021 10:39:34 +0000 Subject: [PATCH] Change refernces to loop state records Resolve issue with OTP 22 performance https://github.com/martinsumner/leveled/issues/326 - by changing refernces to loop state. The test perf_SUITE proves the issue. OTP 22, without fixes: Fold pre-close 41209 ms post-close 688 ms OTP 22, with fixes: Fold pre-close 401 ms post-close 317 ms --- src/leveled_penciller.erl | 16 +++++-- test/end_to_end/perf_SUITE.erl | 85 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 test/end_to_end/perf_SUITE.erl diff --git a/src/leveled_penciller.erl b/src/leveled_penciller.erl index c80f3ce3..4e972df1 100644 --- a/src/leveled_penciller.erl +++ b/src/leveled_penciller.erl @@ -796,9 +796,15 @@ handle_call({fetch_keys, [State#state.levelzero_size], SW, 0.01), + + %% Rename any reference to loop state that may be used by the function + %% to be returned - https://github.com/martinsumner/leveled/issues/326 + Manifest = State#state.manifest, + SnapshotTime = State#state.snapshot_time, + SetupFoldFun = fun(Level, Acc) -> - Pointers = leveled_pmanifest:range_lookup(State#state.manifest, + Pointers = leveled_pmanifest:range_lookup(Manifest, Level, StartKey, EndKey), @@ -812,7 +818,7 @@ handle_call({fetch_keys, fun() -> keyfolder({FilteredL0, SSTiter}, {StartKey, EndKey}, - {AccFun, InitAcc, State#state.snapshot_time}, + {AccFun, InitAcc, SnapshotTime}, {SegmentList, LastModRange0, MaxKeys}) end, case By of @@ -1403,9 +1409,9 @@ roll_memory(State, true) -> ManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1, RootPath = sst_rootpath(State#state.root_path), FileName = sst_filename(ManSQN, 0, 0), - FetchFun = fun(Slot) -> lists:nth(Slot, State#state.levelzero_cache) end, - KVList = leveled_pmem:to_list(length(State#state.levelzero_cache), - FetchFun), + LZC = State#state.levelzero_cache, + FetchFun = fun(Slot) -> lists:nth(Slot, LZC) end, + KVList = leveled_pmem:to_list(length(LZC), FetchFun), R = leveled_sst:sst_new(RootPath, FileName, 0, diff --git a/test/end_to_end/perf_SUITE.erl b/test/end_to_end/perf_SUITE.erl new file mode 100644 index 00000000..9ac6f393 --- /dev/null +++ b/test/end_to_end/perf_SUITE.erl @@ -0,0 +1,85 @@ +-module(perf_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include("include/leveled.hrl"). +-export([all/0]). +-export([bigpcl_bucketlist/1 + ]). + +all() -> [bigpcl_bucketlist]. + + +bigpcl_bucketlist(_Config) -> + %% https://github.com/martinsumner/leveled/issues/326 + %% In OTP 22+ there appear to be issues with anonymous functions which + %% have a reference to loop state, requiring a copy of all the loop state + %% to be made when returning the function. + %% This test creates alarge loop state on the leveled_penciller to prove + %% this. + %% The problem can be resolved simply by renaming the element of the loop + %% state using within the anonymous function. + RootPath = testutil:reset_filestructure(), + BucketCount = 500, + ObjectCount = 100, + StartOpts1 = [{root_path, RootPath}, + {max_journalsize, 50000000}, + {cache_size, 4000}, + {max_pencillercachesize, 128000}, + {max_sstslots, 256}, + {sync_strategy, testutil:sync_strategy()}, + {compression_point, on_compact}], + {ok, Bookie1} = leveled_bookie:book_start(StartOpts1), + BucketList = + lists:map(fun(I) -> list_to_binary(integer_to_list(I)) end, + lists:seq(1, BucketCount)), + + MapFun = + fun(B) -> + testutil:generate_objects(ObjectCount, 1, [], + leveled_rand:rand_bytes(100), + fun() -> [] end, + B) + end, + ObjLofL = lists:map(MapFun, BucketList), + lists:foreach(fun(ObjL) -> testutil:riakload(Bookie1, ObjL) end, ObjLofL), + BucketFold = + fun(B, _K, _V, Acc) -> + case sets:is_element(B, Acc) of + true -> + Acc; + false -> + sets:add_element(B, Acc) + end + end, + FBAccT = {BucketFold, sets:new()}, + + {async, BucketFolder1} = + leveled_bookie:book_headfold(Bookie1, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, false, false), + + {FoldTime1, BucketList1} = timer:tc(BucketFolder1, []), + true = BucketCount == sets:size(BucketList1), + ok = leveled_bookie:book_close(Bookie1), + + {ok, Bookie2} = leveled_bookie:book_start(StartOpts1), + + {async, BucketFolder2} = + leveled_bookie:book_headfold(Bookie2, + ?RIAK_TAG, + {bucket_list, BucketList}, + FBAccT, + false, false, false), + {FoldTime2, BucketList2} = timer:tc(BucketFolder2, []), + true = BucketCount == sets:size(BucketList2), + + io:format("Fold pre-close ~w ms post-close ~w ms~n", + [FoldTime1 div 1000, FoldTime2 div 1000]), + + true = FoldTime1 < 10 * FoldTime2, + %% The fold in-memory should be the same order of magnitude of response + %% time as the fold post-persistence + + ok = leveled_bookie:book_destroy(Bookie2). +