From 263c29e514e87a856949e7f43e813172d3678d16 Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov Date: Sun, 28 Jan 2024 03:32:54 +0000 Subject: [PATCH] tests: add helpers for generating entry slices Signed-off-by: Pavel Kalinnikov --- log_test.go | 224 ++++++++++++++++++++++++---------------------------- 1 file changed, 105 insertions(+), 119 deletions(-) diff --git a/log_test.go b/log_test.go index 2711ff9c..1260a84b 100644 --- a/log_test.go +++ b/log_test.go @@ -24,26 +24,26 @@ import ( ) func TestFindConflict(t *testing.T) { - previousEnts := []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}} + previousEnts := index(1).terms(1, 2, 3) tests := []struct { ents []pb.Entry wconflict uint64 }{ // no conflict, empty ent - {[]pb.Entry{}, 0}, + {nil, 0}, // no conflict - {[]pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}}, 0}, - {[]pb.Entry{{Index: 2, Term: 2}, {Index: 3, Term: 3}}, 0}, - {[]pb.Entry{{Index: 3, Term: 3}}, 0}, + {index(1).terms(1, 2, 3), 0}, + {index(2).terms(2, 3), 0}, + {index(3).terms(3), 0}, // no conflict, but has new entries - {[]pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 4}}, 4}, - {[]pb.Entry{{Index: 2, Term: 2}, {Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 4}}, 4}, - {[]pb.Entry{{Index: 3, Term: 3}, {Index: 4, Term: 4}, {Index: 5, Term: 4}}, 4}, - {[]pb.Entry{{Index: 4, Term: 4}, {Index: 5, Term: 4}}, 4}, + {index(1).terms(1, 2, 3, 4, 4), 4}, + {index(2).terms(2, 3, 4, 5), 4}, + {index(3).terms(3, 4, 4), 4}, + {index(4).terms(4, 4), 4}, // conflicts with existing entries - {[]pb.Entry{{Index: 1, Term: 4}, {Index: 2, Term: 4}}, 1}, - {[]pb.Entry{{Index: 2, Term: 1}, {Index: 3, Term: 4}, {Index: 4, Term: 4}}, 2}, - {[]pb.Entry{{Index: 3, Term: 1}, {Index: 4, Term: 2}, {Index: 5, Term: 4}, {Index: 6, Term: 4}}, 3}, + {index(1).terms(4, 4), 1}, + {index(2).terms(1, 4, 4), 2}, + {index(3).terms(1, 2, 4, 4), 3}, } for i, tt := range tests { @@ -56,13 +56,6 @@ func TestFindConflict(t *testing.T) { } func TestFindConflictByTerm(t *testing.T) { - ents := func(fromIndex uint64, terms []uint64) []pb.Entry { - e := make([]pb.Entry, 0, len(terms)) - for i, term := range terms { - e = append(e, pb.Entry{Term: term, Index: fromIndex + uint64(i)}) - } - return e - } for _, tt := range []struct { ents []pb.Entry // ents[0] contains the (index, term) of the snapshot index uint64 @@ -70,28 +63,28 @@ func TestFindConflictByTerm(t *testing.T) { want uint64 }{ // Log starts from index 1. - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 100, term: 2, want: 100}, // ErrUnavailable - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 5, term: 6, want: 5}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 5, term: 5, want: 5}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 5, term: 4, want: 2}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 5, term: 2, want: 2}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 5, term: 1, want: 0}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 1, term: 2, want: 1}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 1, term: 1, want: 0}, - {ents: ents(0, []uint64{0, 2, 2, 5, 5, 5}), index: 0, term: 0, want: 0}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 100, term: 2, want: 100}, // ErrUnavailable + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 6, want: 5}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 5, want: 5}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 4, want: 2}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 2, want: 2}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 5, term: 1, want: 0}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 1, term: 2, want: 1}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 1, term: 1, want: 0}, + {ents: index(0).terms(0, 2, 2, 5, 5, 5), index: 0, term: 0, want: 0}, // Log with compacted entries. - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 30, term: 3, want: 30}, // ErrUnavailable - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 14, term: 9, want: 14}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 14, term: 4, want: 14}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 14, term: 3, want: 12}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 14, term: 2, want: 9}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 11, term: 5, want: 11}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 10, term: 5, want: 10}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 10, term: 3, want: 10}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 10, term: 2, want: 9}, - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 9, term: 2, want: 9}, // ErrCompacted - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 4, term: 2, want: 4}, // ErrCompacted - {ents: ents(10, []uint64{3, 3, 3, 4, 4, 4}), index: 0, term: 0, want: 0}, // ErrCompacted + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 30, term: 3, want: 30}, // ErrUnavailable + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 9, want: 14}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 4, want: 14}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 3, want: 12}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 14, term: 2, want: 9}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 11, term: 5, want: 11}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 5, want: 10}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 3, want: 10}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 10, term: 2, want: 9}, + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 9, term: 2, want: 9}, // ErrCompacted + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 4, term: 2, want: 4}, // ErrCompacted + {ents: index(10).terms(3, 3, 3, 4, 4, 4), index: 0, term: 0, want: 0}, // ErrCompacted } { t.Run("", func(t *testing.T) { st := NewMemoryStorage() @@ -113,7 +106,7 @@ func TestFindConflictByTerm(t *testing.T) { } func TestIsUpToDate(t *testing.T) { - previousEnts := []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}} + previousEnts := index(1).terms(1, 2, 3) raftLog := newLog(NewMemoryStorage(), raftLogger) raftLog.append(previousEnts...) tests := []struct { @@ -143,7 +136,7 @@ func TestIsUpToDate(t *testing.T) { } func TestAppend(t *testing.T) { - previousEnts := []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}} + previousEnts := index(1).terms(1, 2) tests := []struct { ents []pb.Entry windex uint64 @@ -151,29 +144,29 @@ func TestAppend(t *testing.T) { wunstable uint64 }{ { - []pb.Entry{}, + nil, 2, - []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}}, + index(1).terms(1, 2), 3, }, { - []pb.Entry{{Index: 3, Term: 2}}, + index(3).terms(2), 3, - []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 2}}, + index(1).terms(1, 2, 2), 3, }, // conflicts with index 1 { - []pb.Entry{{Index: 1, Term: 2}}, + index(1).terms(2), 1, - []pb.Entry{{Index: 1, Term: 2}}, + index(1).terms(2), 1, }, // conflicts with index 2 { - []pb.Entry{{Index: 2, Term: 3}, {Index: 3, Term: 3}}, + index(2).terms(3, 3), 3, - []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 3}, {Index: 3, Term: 3}}, + index(1).terms(1, 3, 3), 2, }, } @@ -203,7 +196,7 @@ func TestAppend(t *testing.T) { // // return false func TestLogMaybeAppend(t *testing.T) { - previousEnts := []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}} + previousEnts := index(1).terms(1, 2, 3) lastindex := uint64(3) lastterm := uint64(3) commit := uint64(1) @@ -221,12 +214,12 @@ func TestLogMaybeAppend(t *testing.T) { }{ // not match: term is different { - lastterm - 1, lastindex, lastindex, []pb.Entry{{Index: lastindex + 1, Term: 4}}, + lastterm - 1, lastindex, lastindex, index(lastindex + 1).terms(4), 0, false, commit, false, }, // not match: index out of bound { - lastterm, lastindex + 1, lastindex, []pb.Entry{{Index: lastindex + 2, Term: 4}}, + lastterm, lastindex + 1, lastindex, index(lastindex + 2).terms(4), 0, false, commit, false, }, // match with the last existing entry @@ -251,36 +244,36 @@ func TestLogMaybeAppend(t *testing.T) { 0, true, commit, false, // commit do not decrease }, { - lastterm, lastindex, lastindex, []pb.Entry{{Index: lastindex + 1, Term: 4}}, + lastterm, lastindex, lastindex, index(lastindex + 1).terms(4), lastindex + 1, true, lastindex, false, }, { - lastterm, lastindex, lastindex + 1, []pb.Entry{{Index: lastindex + 1, Term: 4}}, + lastterm, lastindex, lastindex + 1, index(lastindex + 1).terms(4), lastindex + 1, true, lastindex + 1, false, }, { - lastterm, lastindex, lastindex + 2, []pb.Entry{{Index: lastindex + 1, Term: 4}}, + lastterm, lastindex, lastindex + 2, index(lastindex + 1).terms(4), lastindex + 1, true, lastindex + 1, false, // do not increase commit higher than lastnewi }, { - lastterm, lastindex, lastindex + 2, []pb.Entry{{Index: lastindex + 1, Term: 4}, {Index: lastindex + 2, Term: 4}}, + lastterm, lastindex, lastindex + 2, index(lastindex+1).terms(4, 4), lastindex + 2, true, lastindex + 2, false, }, // match with the entry in the middle { - lastterm - 1, lastindex - 1, lastindex, []pb.Entry{{Index: lastindex, Term: 4}}, + lastterm - 1, lastindex - 1, lastindex, index(lastindex).terms(4), lastindex, true, lastindex, false, }, { - lastterm - 2, lastindex - 2, lastindex, []pb.Entry{{Index: lastindex - 1, Term: 4}}, + lastterm - 2, lastindex - 2, lastindex, index(lastindex - 1).terms(4), lastindex - 1, true, lastindex - 1, false, }, { - lastterm - 3, lastindex - 3, lastindex, []pb.Entry{{Index: lastindex - 2, Term: 4}}, + lastterm - 3, lastindex - 3, lastindex, index(lastindex - 2).terms(4), lastindex - 2, true, lastindex - 2, true, // conflict with existing committed entry }, { - lastterm - 2, lastindex - 2, lastindex, []pb.Entry{{Index: lastindex - 1, Term: 4}, {Index: lastindex, Term: 4}}, + lastterm - 2, lastindex - 2, lastindex, index(lastindex-1).terms(4, 4), lastindex, true, lastindex, false, }, } @@ -312,25 +305,24 @@ func TestLogMaybeAppend(t *testing.T) { // TestCompactionSideEffects ensures that all the log related functionality works correctly after // a compaction. func TestCompactionSideEffects(t *testing.T) { - var i uint64 // Populate the log with 1000 entries; 750 in stable storage and 250 in unstable. lastIndex := uint64(1000) unstableIndex := uint64(750) lastTerm := lastIndex storage := NewMemoryStorage() - for i = 1; i <= unstableIndex; i++ { - storage.Append([]pb.Entry{{Term: i, Index: i}}) - } + require.NoError(t, storage.Append(index(1).to(unstableIndex, func(index uint64) uint64 { + return index + }))) raftLog := newLog(storage, raftLogger) - for i = unstableIndex; i < lastIndex; i++ { - raftLog.append(pb.Entry{Term: i + 1, Index: i + 1}) - } + raftLog.append(index(unstableIndex+1).to(lastIndex, func(index uint64) uint64 { + return index + })...) require.True(t, raftLog.maybeCommit(lastIndex, lastTerm)) raftLog.appliedTo(raftLog.committed, 0 /* size */) offset := uint64(500) - storage.Compact(offset) + require.NoError(t, storage.Compact(offset)) require.Equal(t, lastIndex, raftLog.lastIndex()) for j := offset; j <= raftLog.lastIndex(); j++ { @@ -346,7 +338,7 @@ func TestCompactionSideEffects(t *testing.T) { require.Equal(t, uint64(751), unstableEnts[0].Index) prev := raftLog.lastIndex() - raftLog.append(pb.Entry{Index: raftLog.lastIndex() + 1, Term: raftLog.lastIndex() + 1}) + raftLog.append(index(raftLog.lastIndex() + 1).terms(raftLog.lastIndex() + 1)...) require.Equal(t, prev+1, raftLog.lastIndex()) ents, err := raftLog.entries(raftLog.lastIndex(), noLimit) @@ -358,11 +350,7 @@ func TestHasNextCommittedEnts(t *testing.T) { snap := pb.Snapshot{ Metadata: pb.SnapshotMetadata{Term: 1, Index: 3}, } - ents := []pb.Entry{ - {Term: 1, Index: 4}, - {Term: 1, Index: 5}, - {Term: 1, Index: 6}, - } + ents := index(4).terms(1, 1, 1) tests := []struct { applied uint64 applying uint64 @@ -416,11 +404,7 @@ func TestNextCommittedEnts(t *testing.T) { snap := pb.Snapshot{ Metadata: pb.SnapshotMetadata{Term: 1, Index: 3}, } - ents := []pb.Entry{ - {Term: 1, Index: 4}, - {Term: 1, Index: 5}, - {Term: 1, Index: 6}, - } + ents := index(4).terms(1, 1, 1) tests := []struct { applied uint64 applying uint64 @@ -475,11 +459,7 @@ func TestAcceptApplying(t *testing.T) { snap := pb.Snapshot{ Metadata: pb.SnapshotMetadata{Term: 1, Index: 3}, } - ents := []pb.Entry{ - {Term: 1, Index: 4}, - {Term: 1, Index: 5}, - {Term: 1, Index: 6}, - } + ents := index(4).terms(1, 1, 1) tests := []struct { index uint64 allowUnstable bool @@ -530,11 +510,7 @@ func TestAppliedTo(t *testing.T) { snap := pb.Snapshot{ Metadata: pb.SnapshotMetadata{Term: 1, Index: 3}, } - ents := []pb.Entry{ - {Term: 1, Index: 4}, - {Term: 1, Index: 5}, - {Term: 1, Index: 6}, - } + ents := index(4).terms(1, 1, 1) tests := []struct { index uint64 size entryEncodingSize @@ -580,7 +556,7 @@ func TestAppliedTo(t *testing.T) { // TestNextUnstableEnts ensures unstableEntries returns the unstable part of the // entries correctly. func TestNextUnstableEnts(t *testing.T) { - previousEnts := []pb.Entry{{Term: 1, Index: 1}, {Term: 2, Index: 2}} + previousEnts := index(1).terms(1, 2) tests := []struct { unstable uint64 wents []pb.Entry @@ -610,7 +586,7 @@ func TestNextUnstableEnts(t *testing.T) { } func TestCommitTo(t *testing.T) { - previousEnts := []pb.Entry{{Term: 1, Index: 1}, {Term: 2, Index: 2}, {Term: 3, Index: 3}} + previousEnts := index(1).terms(1, 2, 3) commit := uint64(2) tests := []struct { commit uint64 @@ -675,13 +651,13 @@ func TestStableToWithSnap(t *testing.T) { {snapi, snapt + 1, nil, snapi + 1}, {snapi - 1, snapt + 1, nil, snapi + 1}, - {snapi + 1, snapt, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 2}, - {snapi, snapt, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 1}, - {snapi - 1, snapt, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 1}, + {snapi + 1, snapt, index(snapi + 1).terms(snapt), snapi + 2}, + {snapi, snapt, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi - 1, snapt, index(snapi + 1).terms(snapt), snapi + 1}, - {snapi + 1, snapt + 1, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 1}, - {snapi, snapt + 1, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 1}, - {snapi - 1, snapt + 1, []pb.Entry{{Index: snapi + 1, Term: snapt}}, snapi + 1}, + {snapi + 1, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, + {snapi - 1, snapt + 1, index(snapi + 1).terms(snapt), snapi + 1}, } for i, tt := range tests { t.Run(fmt.Sprint(i), func(t *testing.T) { @@ -692,7 +668,6 @@ func TestStableToWithSnap(t *testing.T) { raftLog.stableTo(tt.stablei, tt.stablet) require.Equal(t, tt.wunstable, raftLog.unstable.offset) }) - } } @@ -719,9 +694,9 @@ func TestCompaction(t *testing.T) { } }() storage := NewMemoryStorage() - for i := uint64(1); i <= tt.lastIndex; i++ { - storage.Append([]pb.Entry{{Index: i}}) - } + require.NoError(t, storage.Append(index(1).to(tt.lastIndex, func(index uint64) uint64 { + return 0 // TODO(pav-kv): this should not be has been zero + }))) raftLog := newLog(storage, raftLogger) raftLog.maybeCommit(tt.lastIndex, 0) @@ -734,7 +709,6 @@ func TestCompaction(t *testing.T) { } require.Equal(t, tt.wleft[j], len(raftLog.allEntries())) } - }) } } @@ -760,9 +734,9 @@ func TestIsOutOfBounds(t *testing.T) { storage := NewMemoryStorage() storage.ApplySnapshot(pb.Snapshot{Metadata: pb.SnapshotMetadata{Index: offset}}) l := newLog(storage, raftLogger) - for i := uint64(1); i <= num; i++ { - l.append(pb.Entry{Index: i + offset}) - } + l.append(index(offset+1).to(offset+num, func(index uint64) uint64 { + return 0 // TODO(pav-kv): this should not has been zero + })...) first := offset + 1 tests := []struct { @@ -834,9 +808,9 @@ func TestTerm(t *testing.T) { storage := NewMemoryStorage() storage.ApplySnapshot(pb.Snapshot{Metadata: pb.SnapshotMetadata{Index: offset, Term: 1}}) l := newLog(storage, raftLogger) - for i := uint64(1); i < num; i++ { - l.append(pb.Entry{Index: offset + i, Term: i}) - } + l.append(index(offset+1).to(offset+num-1, func(index uint64) uint64 { + return index - offset + })...) for i, tt := range []struct { idx uint64 @@ -897,11 +871,7 @@ func TestSlice(t *testing.T) { halfe := pb.Entry{Index: half, Term: half} entries := func(from, to uint64) []pb.Entry { - res := make([]pb.Entry, 0, to-from) - for i := from; i < to; i++ { - res = append(res, pb.Entry{Index: i, Term: i}) - } - return res + return index(from).to(to-1, func(index uint64) uint64 { return index }) } storage := NewMemoryStorage() @@ -986,11 +956,7 @@ func TestScan(t *testing.T) { last := offset + num half := offset + num/2 entries := func(from, to uint64) []pb.Entry { - res := make([]pb.Entry, 0, to-from) - for i := from; i < to; i++ { - res = append(res, pb.Entry{Index: i, Term: i}) - } - return res + return index(from).to(to-1, func(index uint64) uint64 { return index }) } entrySize := entsSize(entries(half, half+1)) @@ -1044,3 +1010,23 @@ func mustTerm(term uint64, err error) uint64 { } return term } + +type index uint64 + +func (i index) terms(terms ...uint64) []pb.Entry { + index := uint64(i) + entries := make([]pb.Entry, 0, len(terms)) + for _, term := range terms { + entries = append(entries, pb.Entry{Term: term, Index: index}) + index++ + } + return entries +} + +func (i index) to(last uint64, term func(index uint64) uint64) []pb.Entry { + entries := make([]pb.Entry, 0, last+1-uint64(i)) + for index := uint64(i); index <= last; index++ { + entries = append(entries, pb.Entry{Term: term(index), Index: index}) + } + return entries +}