From 770dfaee5cd6c283e2893432e0cc3e00f1507180 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Wed, 31 Jul 2024 18:18:13 +0000 Subject: [PATCH 1/2] kvserver: split snapshot SSTables for mvcc keys into multiple SSTs Previously, we'd only create one sstable for all mvcc keys in a range when ingesting a rebalance/recovery snapshot into Pebble. This increased write-amp in Pebble as more sstables would have to be compacted into it (or the sstable then split into smaller ones in Pebble), and had other consequences such as massive filter blocks in the large singular sstable. This change adds a new cluster setting, kv.snapshot_rebalance.max_sst_size, that sets the max size of the sstables containing user/mvcc keys in a range. If an sstable exceeds this size in multiSSTWriter, we roll over that sstable and create a new one. Epic: CRDB-8471 Fixes: #67284 Release note (performance improvement): Reduce the write-amplification impact of rebalances by splitting snapshot sstable files into smaller ones before ingesting them into Pebble. --- pkg/kv/kvserver/BUILD.bazel | 1 + .../replica_sst_snapshot_storage_test.go | 127 ++++++- pkg/kv/kvserver/store_snapshot.go | 314 ++++++++++++++---- pkg/storage/engine_key.go | 5 + pkg/storage/sst_writer.go | 11 + 5 files changed, 391 insertions(+), 67 deletions(-) diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index da3adba19042..15b6e9ad50a9 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -528,6 +528,7 @@ go_test( "@com_github_cockroachdb_errors//oserror", "@com_github_cockroachdb_logtags//:logtags", "@com_github_cockroachdb_pebble//:pebble", + "@com_github_cockroachdb_pebble//sstable", "@com_github_cockroachdb_pebble//vfs", "@com_github_cockroachdb_redact//:redact", "@com_github_dustin_go_humanize//:go-humanize", diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go index 0ded1aa2fc51..6e13695a7e98 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go @@ -31,6 +31,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/cockroachdb/errors/oserror" + "github.com/cockroachdb/pebble/sstable" + "github.com/cockroachdb/pebble/vfs" "github.com/stretchr/testify/require" "golang.org/x/time/rate" ) @@ -276,10 +278,12 @@ func TestMultiSSTWriterInitSST(t *testing.T) { EndKey: roachpb.RKeyMax, } keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] msstw, err := newMultiSSTWriter( - ctx, cluster.MakeTestingClusterSettings(), scratch, keySpans, 0, - false, /* skipRangeDelForLastSpan */ + ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, + false, /* skipRangeDelForMVCCSpan */ ) require.NoError(t, err) _, err = msstw.Finish(ctx) @@ -315,6 +319,117 @@ func TestMultiSSTWriterInitSST(t *testing.T) { } } +func buildIterForScratch( + t *testing.T, keySpans []roachpb.Span, scratch *SSTSnapshotStorageScratch, +) (storage.MVCCIterator, error) { + var openFiles []sstable.ReadableFile + for _, sstPath := range scratch.SSTs()[len(keySpans)-1:] { + f, err := vfs.Default.Open(sstPath) + require.NoError(t, err) + openFiles = append(openFiles, f) + } + mvccSpan := keySpans[len(keySpans)-1] + + return storage.NewSSTIterator([][]sstable.ReadableFile{openFiles}, storage.IterOptions{ + LowerBound: mvccSpan.Key, + UpperBound: mvccSpan.EndKey, + }) +} + +// TestMultiSSTWriterSize tests the effect of lowering the max size +// of sstables in a multiSSTWriter, and ensuring that the produced sstables +// are still correct. +func TestMultiSSTWriterSize(t *testing.T) { + defer leaktest.AfterTest(t)() + + ctx := context.Background() + testRangeID := roachpb.RangeID(1) + testSnapUUID := uuid.Must(uuid.FromBytes([]byte("foobar1234567890"))) + testLimiter := rate.NewLimiter(rate.Inf, 0) + + cleanup, eng := newOnDiskEngine(ctx, t) + defer cleanup() + defer eng.Close() + + sstSnapshotStorage := NewSSTSnapshotStorage(eng, testLimiter) + ref := sstSnapshotStorage.NewScratchSpace(testRangeID, testSnapUUID) + scratch := sstSnapshotStorage.NewScratchSpace(testRangeID, testSnapUUID) + settings := cluster.MakeTestingClusterSettings() + + desc := roachpb.RangeDescriptor{ + StartKey: roachpb.RKey("d"), + EndKey: roachpb.RKeyMax, + } + keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] + + // Make a reference msstw with the default size. + referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false) + require.NoError(t, err) + require.Equal(t, int64(0), referenceMsstw.dataSize) + + for i := range localSpans { + require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) + } + + for i := 0; i < 100; i++ { + require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + } + _, err = referenceMsstw.Finish(ctx) + require.NoError(t, err) + + refIter, err := buildIterForScratch(t, keySpans, ref) + require.NoError(t, err) + defer refIter.Close() + + MaxSnapshotSSTableSize.Override(ctx, &settings.SV, 100) + + multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false) + require.NoError(t, err) + require.Equal(t, int64(0), multiSSTWriter.dataSize) + + for i := range localSpans { + require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) + } + + for i := 0; i < 100; i++ { + require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + } + + _, err = multiSSTWriter.Finish(ctx) + require.NoError(t, err) + require.Greater(t, len(scratch.SSTs()), len(ref.SSTs())) + + iter, err := buildIterForScratch(t, keySpans, scratch) + require.NoError(t, err) + defer iter.Close() + + iter.SeekGE(storage.MVCCKey{Key: mvccSpan.Key}) + refIter.SeekGE(storage.MVCCKey{Key: mvccSpan.Key}) + valid, err := iter.Valid() + valid2, err2 := refIter.Valid() + require.NoError(t, err) + require.NoError(t, err2) + + for valid && valid2 { + + require.Equal(t, iter.UnsafeKey(), refIter.UnsafeKey()) + val, err := iter.UnsafeValue() + require.NoError(t, err) + val2, err2 := refIter.UnsafeValue() + require.NoError(t, err2) + require.Equal(t, val, val2) + iter.Next() + refIter.Next() + valid, err = iter.Valid() + valid2, err2 = refIter.Valid() + require.NoError(t, err) + require.NoError(t, err2) + } + require.Equal(t, valid, valid2) +} + // TestMultiSSTWriterAddLastSpan tests that multiSSTWriter initializes each of // the SST files associated with the replicated key ranges by writing a range // deletion tombstone that spans the entire range of each respectively, except @@ -342,14 +457,16 @@ func TestMultiSSTWriterAddLastSpan(t *testing.T) { EndKey: roachpb.RKeyMax, } keySpans := rditer.MakeReplicatedKeySpans(&desc) + localSpans := keySpans[:len(keySpans)-1] + mvccSpan := keySpans[len(keySpans)-1] msstw, err := newMultiSSTWriter( - ctx, cluster.MakeTestingClusterSettings(), scratch, keySpans, 0, - true, /* skipRangeDelForLastSpan */ + ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, + true, /* skipRangeDelForMVCCSpan */ ) require.NoError(t, err) if addRangeDel { - require.NoError(t, msstw.addRangeDelForLastSpan()) + require.NoError(t, msstw.addClearForMVCCSpan()) } testKey := storage.MVCCKey{Key: roachpb.RKey("d1").AsRawKey(), Timestamp: hlc.Timestamp{WallTime: 1}} testEngineKey, _ := storage.DecodeEngineKey(storage.EncodeMVCCKey(testKey)) diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 997132b4411f..01a34a952033 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -11,7 +11,9 @@ package kvserver import ( + "bytes" "context" + "fmt" "io" "time" @@ -62,6 +64,16 @@ const ( tagSnapshotTiming = "snapshot_timing_tag" ) +// MaxSnapshotSSTableSize is the maximum size of an sstable containing MVCC/user keys +// in a snapshot before we truncate and write a new snapshot sstable. +var MaxSnapshotSSTableSize = settings.RegisterByteSizeSetting( + settings.SystemOnly, + "kv.snapshot_rebalance.max_sst_size", + "maximum size of a rebalance or recovery SST size", + 128<<20, // 128 MB + settings.PositiveInt, +) + // snapshotMetrics contains metrics on the number and size of snapshots in // progress or in the snapshot queue. type snapshotMetrics struct { @@ -115,10 +127,23 @@ type kvBatchSnapshotStrategy struct { // multiSSTWriter is a wrapper around an SSTWriter and SSTSnapshotStorageScratch // that handles chunking SSTs and persisting them to disk. type multiSSTWriter struct { - st *cluster.Settings - scratch *SSTSnapshotStorageScratch - currSST storage.SSTWriter - keySpans []roachpb.Span + st *cluster.Settings + scratch *SSTSnapshotStorageScratch + currSST storage.SSTWriter + // localKeySpans are key spans that are considered unsplittable across sstables, and + // represent the range's range local key spans. In contrast, mvccKeySpan can be split + // across multiple sstables if one of them exceeds maxSSTSize. The expectation is + // that for large ranges, keys in mvccKeySpan will dominate in size compared to keys + // in localKeySpans. + localKeySpans []roachpb.Span + mvccKeySpan roachpb.Span + // mvccSSTSpans reflects the actual split of the mvccKeySpan into constituent + // sstables. + mvccSSTSpans []storage.EngineKeyRange + // currSpan is the index of the current span being written to. The first + // len(localKeySpans) spans are localKeySpans, and the rest are mvccSSTSpans. + // In a sense, currSpan indexes into a slice composed of + // append(localKeySpans, mvccSSTSpans). currSpan int // The approximate size of the SST chunk to buffer in memory on the receiver // before flushing to disk. @@ -128,33 +153,84 @@ type multiSSTWriter struct { dataSize int64 // The total size of the SSTs. sstSize int64 - // if skipRangeDelForLastSpan is true, the last span is not ClearRanged in the - // same sstable. We rely on the caller to take care of clearing this span - // through a different process (eg. IngestAndExcise on pebble). - skipRangeDelForLastSpan bool + // if skipClearForMVCCSpan is true, the MVCC span is not ClearEngineRange()d in + // the same sstable. We rely on the caller to take care of clearing this span + // through a different process (eg. IngestAndExcise on pebble). Note that + // having this bool to true also disables all range key fragmentation + // and splitting of sstables in the mvcc span. + skipClearForMVCCSpan bool + // maxSSTSize is the maximum size to use for SSTs containing MVCC/user keys. + // Once the sstable writer reaches this size, it will be finalized and a new + // sstable will be created. + maxSSTSize int64 + // rangeKeyFrag is used to fragment range keys across the mvcc key spans. + rangeKeyFrag rangekey.Fragmenter } func newMultiSSTWriter( ctx context.Context, st *cluster.Settings, scratch *SSTSnapshotStorageScratch, - keySpans []roachpb.Span, + localKeySpans []roachpb.Span, + mvccKeySpan roachpb.Span, sstChunkSize int64, - skipRangeDelForLastSpan bool, -) (multiSSTWriter, error) { - msstw := multiSSTWriter{ - st: st, - scratch: scratch, - keySpans: keySpans, - sstChunkSize: sstChunkSize, - skipRangeDelForLastSpan: skipRangeDelForLastSpan, + skipClearForMVCCSpan bool, +) (*multiSSTWriter, error) { + msstw := &multiSSTWriter{ + st: st, + scratch: scratch, + localKeySpans: localKeySpans, + mvccKeySpan: mvccKeySpan, + mvccSSTSpans: []storage.EngineKeyRange{{ + Start: storage.EngineKey{Key: mvccKeySpan.Key}, + End: storage.EngineKey{Key: mvccKeySpan.EndKey}, + }}, + sstChunkSize: sstChunkSize, + skipClearForMVCCSpan: skipClearForMVCCSpan, + } + if !skipClearForMVCCSpan { + // If skipClearForMVCCSpan is true, we don't split the MVCC span across + // multiple sstables, as addClearForMVCCSpan could be called by the caller + // at any time. + msstw.maxSSTSize = MaxSnapshotSSTableSize.Get(&st.SV) + } + msstw.rangeKeyFrag = rangekey.Fragmenter{ + Cmp: storage.EngineComparer.Compare, + Format: storage.EngineComparer.FormatKey, + Emit: msstw.emitRangeKey, } + if err := msstw.initSST(ctx); err != nil { return msstw, err } return msstw, nil } +func (msstw *multiSSTWriter) emitRangeKey(key rangekey.Span) { + for i := range key.Keys { + if err := msstw.currSST.PutInternalRangeKey(key.Start, key.End, key.Keys[i]); err != nil { + panic(fmt.Sprintf("failed to put range key in sst: %s", err)) + } + } +} + +// currentSpan returns the current user-provided span that +// is being written to. Note that this does not account for +// mvcc keys being split across multiple sstables. +func (msstw *multiSSTWriter) currentSpan() roachpb.Span { + if msstw.currSpanIsMVCCSpan() { + return msstw.mvccKeySpan + } + return msstw.localKeySpans[msstw.currSpan] +} + +func (msstw *multiSSTWriter) currSpanIsMVCCSpan() bool { + if msstw.currSpan >= len(msstw.localKeySpans)+len(msstw.mvccSSTSpans) { + panic("current span is out of bounds") + } + return msstw.currSpan >= len(msstw.localKeySpans) +} + func (msstw *multiSSTWriter) initSST(ctx context.Context) error { newSSTFile, err := msstw.scratch.NewFile(ctx, msstw.sstChunkSize) if err != nil { @@ -162,26 +238,94 @@ func (msstw *multiSSTWriter) initSST(ctx context.Context) error { } newSST := storage.MakeIngestionSSTWriter(ctx, msstw.st, newSSTFile) msstw.currSST = newSST - if msstw.skipRangeDelForLastSpan && msstw.currSpan == len(msstw.keySpans)-1 { - // Skip this ClearRange, as it will be excised at ingestion time in the - // engine instead. - return nil - } - if err := msstw.currSST.ClearRawRange( - msstw.keySpans[msstw.currSpan].Key, msstw.keySpans[msstw.currSpan].EndKey, - true /* pointKeys */, true, /* rangeKeys */ - ); err != nil { - msstw.currSST.Close() - return errors.Wrap(err, "failed to clear range on sst file writer") + if !msstw.currSpanIsMVCCSpan() || (!msstw.skipClearForMVCCSpan && msstw.currSpan <= len(msstw.localKeySpans)) { + // We're either in a local key span, or we're in the first MVCC sstable + // span (before any splits). Add a RangeKeyDel for the whole span. If this + // is the MVCC span, we don't need to keep re-adding it to the fragmenter + // as the fragmenter will take care of splits. Note that currentSpan() + // will return the entire mvcc span in the case we're at an MVCC span. + startKey := storage.EngineKey{Key: msstw.currentSpan().Key}.Encode() + endKey := storage.EngineKey{Key: msstw.currentSpan().EndKey}.Encode() + trailer := pebble.MakeInternalKeyTrailer(0, pebble.InternalKeyKindRangeKeyDelete) + s := rangekey.Span{Start: startKey, End: endKey, Keys: []rangekey.Key{{Trailer: trailer}}} + msstw.rangeKeyFrag.Add(s) } return nil } -func (msstw *multiSSTWriter) finalizeSST(ctx context.Context) error { +func (msstw *multiSSTWriter) finalizeSST(ctx context.Context, nextKey *storage.EngineKey) error { + currSpan := msstw.currentSpan() + if msstw.currSpanIsMVCCSpan() { + // We're in the MVCC span (ie. MVCC / user keys). If skipClearForMVCCSpan + // is true, we don't write a clearRange for the last span at all. Otherwise, + // we need to write a clearRange for all keys leading up to the current key + // we're writing. + currEngineSpan := msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)] + if !msstw.skipClearForMVCCSpan { + if err := msstw.currSST.ClearEngineRange( + currEngineSpan.Start, currEngineSpan.End, + ); err != nil { + msstw.currSST.Close() + return errors.Wrap(err, "failed to clear range on sst file writer") + } + } + } else { + if err := msstw.currSST.ClearRawRange( + currSpan.Key, currSpan.EndKey, + true /* pointKeys */, false, /* rangeKeys */ + ); err != nil { + msstw.currSST.Close() + return errors.Wrap(err, "failed to clear range on sst file writer") + } + } + + // If we're at the last span, call Finish on the fragmenter. If we're not at the + // last span, call Truncate. + if msstw.currSpan == len(msstw.localKeySpans)+len(msstw.mvccSSTSpans)-1 { + msstw.rangeKeyFrag.Finish() + } else { + endKey := storage.EngineKey{Key: currSpan.EndKey} + if msstw.currSpanIsMVCCSpan() { + endKey = msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)].End + } + msstw.rangeKeyFrag.Truncate(endKey.Encode()) + } + err := msstw.currSST.Finish() if err != nil { return errors.Wrap(err, "failed to finish sst") } + if nextKey != nil { + meta := msstw.currSST.Meta + encodedNextKey := nextKey.Encode() + if meta.HasPointKeys && storage.EngineKeyCompare(meta.LargestPoint.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestPoint.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest point key %s > next sstable start key %s", + meta.LargestPoint.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest point key %s > next sstable start key %s", + metaEndKey, nextKey) + } + if meta.HasRangeDelKeys && storage.EngineKeyCompare(meta.LargestRangeDel.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestRangeDel.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range del %s > next sstable start key %s", + meta.LargestRangeDel.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range del %s > next sstable start key %s", + metaEndKey, nextKey) + } + if meta.HasRangeKeys && storage.EngineKeyCompare(meta.LargestRangeKey.UserKey, encodedNextKey) > 0 { + metaEndKey, ok := storage.DecodeEngineKey(meta.LargestRangeKey.UserKey) + if !ok { + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range key %s > next sstable start key %s", + meta.LargestRangeKey.UserKey, nextKey) + } + return errors.Errorf("multiSSTWriter created overlapping ingestion sstables: sstable largest range key %s > next sstable start key %s", + metaEndKey, nextKey) + } + } msstw.dataSize += msstw.currSST.DataSize msstw.sstSize += int64(msstw.currSST.Meta.Size) msstw.currSpan++ @@ -189,22 +333,22 @@ func (msstw *multiSSTWriter) finalizeSST(ctx context.Context) error { return nil } -// addRangeDelForLastSpan allows us to explicitly add a deletion tombstone -// for the last span in the msstw, if it was instantiated with the expectation +// addClearForMVCCSpan allows us to explicitly add a deletion tombstone +// for the mvcc span in the msstw, if it was instantiated with the expectation // that no tombstone was necessary. -func (msstw *multiSSTWriter) addRangeDelForLastSpan() error { - if !msstw.skipRangeDelForLastSpan { +func (msstw *multiSSTWriter) addClearForMVCCSpan() error { + if !msstw.skipClearForMVCCSpan { // Nothing to do. return nil } - if msstw.currSpan < len(msstw.keySpans)-1 { - // When we switch to the last key span, we will just add a rangedel for it. - // Set skipRangeDelForLastSpan to false. - msstw.skipRangeDelForLastSpan = false + if msstw.currSpan < len(msstw.localKeySpans) { + // When we switch to the mvcc key span, we will just add a rangedel for it. + // Set skipClearForMVCCSpan to false. + msstw.skipClearForMVCCSpan = false return nil } - if msstw.currSpan > len(msstw.keySpans)-1 { - panic("cannot addRangeDel if sst writer has moved past user keys") + if msstw.currSpan >= len(msstw.localKeySpans) { + panic("cannot clearEngineRange if sst writer has moved past user keys") } panic("multiSSTWriter already added keys to sstable that cannot be deleted by a rangedel/rangekeydel within it") } @@ -213,31 +357,57 @@ func (msstw *multiSSTWriter) addRangeDelForLastSpan() error { // writer for writing a point/range key at key. For point keys, endKey and key // must equal each other. func (msstw *multiSSTWriter) rolloverSST( - ctx context.Context, key roachpb.Key, endKey roachpb.Key, + ctx context.Context, key storage.EngineKey, endKey storage.EngineKey, ) error { - for msstw.keySpans[msstw.currSpan].EndKey.Compare(key) <= 0 { + for msstw.currentSpan().EndKey.Compare(key.Key) <= 0 { // Finish the current SST, write to the file, and move to the next key // range. - if err := msstw.finalizeSST(ctx); err != nil { + if err := msstw.finalizeSST(ctx, &key); err != nil { return err } if err := msstw.initSST(ctx); err != nil { return err } } - if msstw.keySpans[msstw.currSpan].Key.Compare(key) > 0 || - msstw.keySpans[msstw.currSpan].EndKey.Compare(endKey) < 0 { - if !key.Equal(endKey) { - return errors.AssertionFailedf("client error: expected %s to fall in one of %s", - roachpb.Span{Key: key, EndKey: endKey}, msstw.keySpans) + currSpan := msstw.currentSpan() + if currSpan.Key.Compare(key.Key) > 0 || currSpan.EndKey.Compare(endKey.Key) < 0 { + if !key.Key.Equal(endKey.Key) { + return errors.AssertionFailedf("client error: expected %s to fall in one of %s or %s", + roachpb.Span{Key: key.Key, EndKey: endKey.Key}, msstw.localKeySpans, msstw.mvccKeySpan) + } + return errors.AssertionFailedf("client error: expected %s to fall in one of %s or %s", key, msstw.localKeySpans, msstw.mvccKeySpan) + } + if msstw.currSpanIsMVCCSpan() && msstw.maxSSTSize > 0 && msstw.currSST.DataSize > msstw.maxSSTSize { + // We're in an MVCC / user keys span, and the current sstable has exceeded + // the max size for MVCC sstables that we should be creating. Split this + // sstable into smaller ones. We do this by splitting the mvccKeySpan + // from [oldStartKey, oldEndKey) to [oldStartKey, key) and [key, oldEndKey). + // The split spans are added to msstw.mvccSSTSpans. + currSpan := &msstw.mvccSSTSpans[msstw.currSpan-len(msstw.localKeySpans)] + if bytes.Equal(currSpan.Start.Key, key.Key) && bytes.Equal(currSpan.Start.Version, key.Version) { + panic("unexpectedly reached max sstable size at start of an mvcc sstable span") + } + oldEndKey := currSpan.End + currSpan.End = key.Copy() + newSpan := storage.EngineKeyRange{Start: currSpan.End, End: oldEndKey} + msstw.mvccSSTSpans = append(msstw.mvccSSTSpans, newSpan) + if msstw.currSpan < len(msstw.localKeySpans)+len(msstw.mvccSSTSpans)-2 { + // This should never happen; we only split sstables when we're at the end + // of mvccSSTSpans. + panic("unexpectedly split an earlier mvcc sstable span in multiSSTWriter") + } + if err := msstw.finalizeSST(ctx, &key); err != nil { + return err + } + if err := msstw.initSST(ctx); err != nil { + return err } - return errors.AssertionFailedf("client error: expected %s to fall in one of %s", key, msstw.keySpans) } return nil } func (msstw *multiSSTWriter) Put(ctx context.Context, key storage.EngineKey, value []byte) error { - if err := msstw.rolloverSST(ctx, key.Key, key.Key); err != nil { + if err := msstw.rolloverSST(ctx, key, key); err != nil { return err } if err := msstw.currSST.PutEngineKey(key, value); err != nil { @@ -253,7 +423,7 @@ func (msstw *multiSSTWriter) PutInternalPointKey( if !ok { return errors.New("cannot decode engine key") } - if err := msstw.rolloverSST(ctx, decodedKey.Key, decodedKey.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedKey, decodedKey); err != nil { return err } var err error @@ -290,11 +460,14 @@ func decodeRangeStartEnd( } func (msstw *multiSSTWriter) PutInternalRangeDelete(ctx context.Context, start, end []byte) error { + if !msstw.skipClearForMVCCSpan { + panic("can only add internal range deletes to multiSSTWriter if skipClearForMVCCSpan is true") + } decodedStart, decodedEnd, err := decodeRangeStartEnd(start, end) if err != nil { return err } - if err := msstw.rolloverSST(ctx, decodedStart.Key, decodedEnd.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedStart, decodedEnd); err != nil { return err } if err := msstw.currSST.ClearRawEncodedRange(start, end); err != nil { @@ -306,11 +479,14 @@ func (msstw *multiSSTWriter) PutInternalRangeDelete(ctx context.Context, start, func (msstw *multiSSTWriter) PutInternalRangeKey( ctx context.Context, start, end []byte, key rangekey.Key, ) error { + if !msstw.skipClearForMVCCSpan { + panic("can only add internal range deletes to multiSSTWriter if skipClearForMVCCSpan is true") + } decodedStart, decodedEnd, err := decodeRangeStartEnd(start, end) if err != nil { return err } - if err := msstw.rolloverSST(ctx, decodedStart.Key, decodedEnd.Key); err != nil { + if err := msstw.rolloverSST(ctx, decodedStart, decodedEnd); err != nil { return err } if err := msstw.currSST.PutInternalRangeKey(start, end, key); err != nil { @@ -325,22 +501,33 @@ func (msstw *multiSSTWriter) PutRangeKey( if start.Compare(end) >= 0 { return errors.AssertionFailedf("start key %s must be before end key %s", end, start) } - if err := msstw.rolloverSST(ctx, start, end); err != nil { + if err := msstw.rolloverSST(ctx, storage.EngineKey{Key: start}, storage.EngineKey{Key: end}); err != nil { return err } - if err := msstw.currSST.PutEngineRangeKey(start, end, suffix, value); err != nil { - return errors.Wrap(err, "failed to put range key in sst") + if msstw.skipClearForMVCCSpan { + // Skip the fragmenter. See the comment in skipClearForMVCCSpan. + if err := msstw.currSST.PutEngineRangeKey(start, end, suffix, value); err != nil { + return errors.Wrap(err, "failed to put range key in sst") + } + return nil } + startKey, endKey := storage.EngineKey{Key: start}.Encode(), storage.EngineKey{Key: end}.Encode() + startTrailer := pebble.MakeInternalKeyTrailer(0, pebble.InternalKeyKindRangeKeySet) + msstw.rangeKeyFrag.Add(rangekey.Span{ + Start: startKey, + End: endKey, + Keys: []rangekey.Key{{Trailer: startTrailer, Suffix: suffix, Value: value}}, + }) return nil } func (msstw *multiSSTWriter) Finish(ctx context.Context) (int64, error) { - if msstw.currSpan < len(msstw.keySpans) { + if msstw.currSpan < (len(msstw.localKeySpans) + len(msstw.mvccSSTSpans)) { for { - if err := msstw.finalizeSST(ctx); err != nil { + if err := msstw.finalizeSST(ctx, nil /* nextKey */); err != nil { return 0, err } - if msstw.currSpan >= len(msstw.keySpans) { + if msstw.currSpan >= (len(msstw.localKeySpans) + len(msstw.mvccSSTSpans)) { break } if err := msstw.initSST(ctx); err != nil { @@ -506,8 +693,11 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( // TODO(aaditya): Remove once we support flushableIngests for shared and // external files in the engine. - skipRangeDelForLastSpan := doExcise && (header.SharedReplicate || header.ExternalReplicate) - msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, keyRanges, kvSS.sstChunkSize, skipRangeDelForLastSpan) + skipClearForMVCCSpan := doExcise && (header.SharedReplicate || header.ExternalReplicate) + // The last key range is the user key span. + localRanges := keyRanges[:len(keyRanges)-1] + mvccRange := keyRanges[len(keyRanges)-1] + msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan) if err != nil { return noSnap, err } @@ -533,7 +723,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( doExcise = false sharedSSTs = nil externalSSTs = nil - if err := msstw.addRangeDelForLastSpan(); err != nil { + if err := msstw.addClearForMVCCSpan(); err != nil { return noSnap, errors.Wrap(err, "adding tombstone for last span") } } @@ -706,7 +896,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( sharedSSTs: sharedSSTs, externalSSTs: externalSSTs, doExcise: doExcise, - includesRangeDelForLastSpan: !skipRangeDelForLastSpan, + includesRangeDelForLastSpan: !skipClearForMVCCSpan, clearedSpans: keyRanges, } diff --git a/pkg/storage/engine_key.go b/pkg/storage/engine_key.go index 08d1b28ee497..2c7bf8748532 100644 --- a/pkg/storage/engine_key.go +++ b/pkg/storage/engine_key.go @@ -419,3 +419,8 @@ func decodeMVCCMetaAndVerify(key roachpb.Key, value []byte) error { } return decodeMVCCValueAndVerify(key, meta.RawBytes) } + +// EngineKeyRange is a key range composed of EngineKeys. +type EngineKeyRange struct { + Start, End EngineKey +} diff --git a/pkg/storage/sst_writer.go b/pkg/storage/sst_writer.go index c4689047f850..7f012a26ba0b 100644 --- a/pkg/storage/sst_writer.go +++ b/pkg/storage/sst_writer.go @@ -298,6 +298,17 @@ func (fw *SSTWriter) ClearEngineRangeKey(start, end roachpb.Key, suffix []byte) return fw.fw.RangeKeyUnset(EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix) } +// ClearEngineRange clears point keys in the specified EngineKey range. +func (fw *SSTWriter) ClearEngineRange(start, end EngineKey) error { + fw.scratch = start.EncodeToBuf(fw.scratch[:0]) + endRaw := end.Encode() + fw.DataSize += int64(len(start.Key)) + int64(len(end.Key)) + if err := fw.fw.DeleteRange(fw.scratch, endRaw); err != nil { + return err + } + return nil +} + // ClearRawEncodedRange implements the InternalWriter interface. func (fw *SSTWriter) ClearRawEncodedRange(start, end []byte) error { startEngine, ok := DecodeEngineKey(start) From 94fb782bed26e1ff420cd400c7b9cd4b7378b64d Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Thu, 15 Aug 2024 16:19:10 -0400 Subject: [PATCH 2/2] kvserver: reenable splitting of snapshot sstables This change updates the snapshot strategy's sender side to iterate over points and ranges together, instead of only iterating on points first, then only ranges. This allows us to more efficiently split snapshot sstables on the receiver side. To avoid the need to add a version gate on the receiver side, we propagate a bool, RangeKeysInOrder, to the receiver which is a signal to it to enable sstable splits. Fixes #129026. Epic: none Release note: None --- pkg/cli/debug.go | 9 ++- pkg/kv/kvserver/client_merge_test.go | 2 +- pkg/kv/kvserver/kvserverpb/raft.proto | 6 ++ pkg/kv/kvserver/rditer/replica_data_iter.go | 37 +++++----- .../kvserver/rditer/replica_data_iter_test.go | 21 +++--- .../TestReplicaDataIterator/r1/all/all/output | 6 +- .../r1/all/exclude-user/output | 4 +- .../r1/all/user-only/output | 6 +- .../r1/replicatedOnly/all/output | 4 +- .../r1/replicatedOnly/exclude-user/output | 2 +- .../r1/replicatedOnly/user-only/output | 2 +- .../TestReplicaDataIterator/r2/all/all/output | 6 +- .../r2/all/exclude-user/output | 4 +- .../r2/all/user-only/output | 6 +- .../r2/replicatedOnly/all/output | 4 +- .../r2/replicatedOnly/exclude-user/output | 2 +- .../r2/replicatedOnly/user-only/output | 2 +- .../TestReplicaDataIterator/r3/all/all/output | 6 +- .../r3/all/exclude-user/output | 4 +- .../r3/all/user-only/output | 6 +- .../r3/replicatedOnly/all/output | 4 +- .../r3/replicatedOnly/exclude-user/output | 2 +- .../r3/replicatedOnly/user-only/output | 2 +- pkg/kv/kvserver/replica_command.go | 1 + pkg/kv/kvserver/replica_learner_test.go | 14 ++-- .../replica_sst_snapshot_storage_test.go | 40 ++++++++-- pkg/kv/kvserver/replica_test.go | 3 +- pkg/kv/kvserver/store_snapshot.go | 74 +++++++++++-------- 28 files changed, 157 insertions(+), 122 deletions(-) diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index ded425bad9f1..eb7d6b0bc278 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -515,10 +515,10 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error { var results int return rditer.IterateReplicaKeySpans(cmd.Context(), &desc, snapshot, debugCtx.replicated, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + if hasPoint { key, err := iter.UnsafeEngineKey() if err != nil { return err @@ -532,8 +532,9 @@ func runDebugRangeData(cmd *cobra.Command, args []string) error { if results == debugCtx.maxResults { return iterutil.StopIteration() } + } - case storage.IterKeyTypeRangesOnly: + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() if err != nil { return err diff --git a/pkg/kv/kvserver/client_merge_test.go b/pkg/kv/kvserver/client_merge_test.go index d2ac16f99c45..f4499612c775 100644 --- a/pkg/kv/kvserver/client_merge_test.go +++ b/pkg/kv/kvserver/client_merge_test.go @@ -3910,7 +3910,7 @@ func TestStoreRangeMergeRaftSnapshot(t *testing.T) { err := rditer.IterateReplicaKeySpans( context.Background(), inSnap.Desc, sendingEngSnapshot, true /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { fw, ok := sstFileWriters[string(span.Key)] if !ok || !fw.span.Equal(span) { return errors.Errorf("unexpected span %s", span) diff --git a/pkg/kv/kvserver/kvserverpb/raft.proto b/pkg/kv/kvserver/kvserverpb/raft.proto index c920a21df903..3aafaf6db777 100644 --- a/pkg/kv/kvserver/kvserverpb/raft.proto +++ b/pkg/kv/kvserver/kvserverpb/raft.proto @@ -211,6 +211,12 @@ message SnapshotRequest { // file contents. bool external_replicate = 13; + // If true, the snapshot is iterating over range keys and point + // keys in key order, as opposed to iterating over point keys first + // and range keys second. The receiver can take advantage of this + // to split points/range keys into multiple sstables for ingestion. + bool range_keys_in_order = 14; + reserved 1, 4, 6, 7, 8, 9; } diff --git a/pkg/kv/kvserver/rditer/replica_data_iter.go b/pkg/kv/kvserver/rditer/replica_data_iter.go index 9dbedf688194..9b51d7b5c6ca 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter.go @@ -404,7 +404,7 @@ func IterateReplicaKeySpans( reader storage.Reader, replicatedOnly bool, replicatedSpansFilter ReplicatedSpansFilter, - visitor func(storage.EngineIterator, roachpb.Span, storage.IterKeyType) error, + visitor func(storage.EngineIterator, roachpb.Span) error, ) error { if !reader.ConsistentIterators() { panic("reader must provide consistent iterators") @@ -426,28 +426,25 @@ func IterateReplicaKeySpans( UnreplicatedByRangeID: true, }) } - keyTypes := []storage.IterKeyType{storage.IterKeyTypePointsOnly, storage.IterKeyTypeRangesOnly} for _, span := range spans { - for _, keyType := range keyTypes { - err := func() error { - iter, err := reader.NewEngineIterator(ctx, storage.IterOptions{ - KeyTypes: keyType, - LowerBound: span.Key, - UpperBound: span.EndKey, - }) - if err != nil { - return err - } - defer iter.Close() - ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key}) - if err == nil && ok { - err = visitor(iter, span, keyType) - } - return err - }() + err := func() error { + iter, err := reader.NewEngineIterator(ctx, storage.IterOptions{ + KeyTypes: storage.IterKeyTypePointsAndRanges, + LowerBound: span.Key, + UpperBound: span.EndKey, + }) if err != nil { - return iterutil.Map(err) + return err } + defer iter.Close() + ok, err := iter.SeekEngineKeyGE(storage.EngineKey{Key: span.Key}) + if err == nil && ok { + err = visitor(iter, span) + } + return err + }() + if err != nil { + return iterutil.Map(err) } } return nil diff --git a/pkg/kv/kvserver/rditer/replica_data_iter_test.go b/pkg/kv/kvserver/rditer/replica_data_iter_test.go index 859478669deb..a1de1020650e 100644 --- a/pkg/kv/kvserver/rditer/replica_data_iter_test.go +++ b/pkg/kv/kvserver/rditer/replica_data_iter_test.go @@ -175,7 +175,7 @@ func verifyIterateReplicaKeySpans( require.NoError(t, IterateReplicaKeySpans(context.Background(), desc, readWriter, replicatedOnly, replicatedSpansFilter, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { // Span should not be empty. @@ -186,8 +186,8 @@ func verifyIterateReplicaKeySpans( require.True(t, span.ContainsKey(key.Key), "%s not in %s", key, span) require.True(t, key.IsLockTableKey() || key.IsMVCCKey(), "%s neither lock nor MVCC", key) - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + if hasPoint { var mvccKey storage.MVCCKey if key.IsMVCCKey() { var err error @@ -213,8 +213,8 @@ func verifyIterateReplicaKeySpans( fmt.Sprintf("%x", key.Version), mvccKey.String(), }) - - case storage.IterKeyTypeRangesOnly: + } + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() require.NoError(t, err) require.True(t, span.Contains(bounds), "%s not contained in %s", bounds, span) @@ -234,9 +234,6 @@ func verifyIterateReplicaKeySpans( mvccRangeKey.String(), }) } - - default: - t.Fatalf("unexpected key type %v", keyType) } } return err @@ -482,9 +479,11 @@ func TestReplicaDataIteratorGlobalRangeKey(t *testing.T) { var actualSpans []roachpb.Span require.NoError(t, IterateReplicaKeySpans( context.Background(), &desc, snapshot, replicatedOnly, ReplicatedSpansAll, - func(iter storage.EngineIterator, span roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, span roachpb.Span) error { // We should never see any point keys. - require.Equal(t, storage.IterKeyTypeRangesOnly, keyType) + hasPoint, hasRange := iter.HasPointAndRange() + require.False(t, hasPoint) + require.True(t, hasRange) // The iterator should already be positioned on the range key, which should // span the entire key span and be the only range key. @@ -590,7 +589,7 @@ func benchReplicaEngineDataIterator(b *testing.B, numRanges, numKeysPerRange, va for _, desc := range descs { err := IterateReplicaKeySpans( context.Background(), &desc, snapshot, false /* replicatedOnly */, ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, _ storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { _, _ = iter.UnsafeEngineKey() diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output index a70c35c621b5..650a35b2e8bd 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{a"-b"} | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor | | /Local/Lock"{a"-b"} | 017a6b12610001 | | 030ce61c175eb445878c36dcf4062ada4c | "a" | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output index d4751da1f82d..89612c49d946 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output index 21d8f0604597..bef327ba62df 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{u""-v""} | 0169897572667462 | | | /Local/RangeID/1/u/RangeTombstone | | /Local/RangeID/1/{u""-v""} | 0169897572667468 | | | /Local/RangeID/1/u/RaftHardState | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000001 | | | /Local/RangeID/1/u/RaftLog/logIndex:1 | | /Local/RangeID/1/{u""-v""} | 016989757266746c0000000000000002 | | | /Local/RangeID/1/u/RaftLog/logIndex:2 | | /Local/RangeID/1/{u""-v""} | 01698975726c7274 | | | /Local/RangeID/1/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/1/{u""-v""} | 016989753a61 | 016989753a78 | 000000000000000109 | /Local/RangeID/1/u":{a"-x"}/0.000000001,0 | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output index b487074ffe65..107ed05a398c 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{a"-b"} | 016b1261ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"a\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{a"-b"} | 017a6b12016b126100ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"a"/RangeDescriptor | | /Local/Lock"{a"-b"} | 017a6b12610001 | | 030ce61c175eb445878c36dcf4062ada4c | "a" | +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output index fc247f4b9f6f..0c6052519a70 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/1/{r""-s""} | 016989726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/1/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/1/{r""-s""} | 016989726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/1/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/1/{r""-s""} | 016989726c67632d | | | /Local/RangeID/1/r/RangeGCThreshold | | /Local/RangeID/1/{r""-s""} | 016989727261736b | | | /Local/RangeID/1/r/RangeAppliedState | | /Local/RangeID/1/{r""-s""} | 01698972726c6c2d | | | /Local/RangeID/1/r/RangeLease | -| /Local/RangeID/1/{r""-s""} | 016989723a61 | 016989723a78 | 000000000000000109 | /Local/RangeID/1/r":{a"-x"}/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000172647363 | | 0000000000000001 | /Local/Range"a"/RangeDescriptor/0.000000001,0 | | /Local/Range"{a"-b"} | 016b1261000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"a"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{a"-b"} | 016b126100ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"a\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output index ffd95d052634..a6b80632c58c 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r1/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | | {a-b} | 61 | | 0000000000000001 | "a"/0.000000001,0 | | {a-b} | 61ffffffff | | 0000000000000001 | "a\xff\xff\xff\xff"/0.000000001,0 | -| {a-b} | 61 | 62 | 000000000000000109 | {a-b}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output index 1c3cb3da75b9..c4adc4e47272 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{b"-c"} | 016b1262ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"b\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{b"-c"} | 017a6b12016b126200ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"b"/RangeDescriptor | | /Local/Lock"{b"-c"} | 017a6b12620001 | | 030ce61c175eb445878c36dcf4062ada4c | "b" | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output index 936fb93c42d1..29775b06a649 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output index c8f0a593037e..c518246b0aa7 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{u""-v""} | 01698a7572667462 | | | /Local/RangeID/2/u/RangeTombstone | | /Local/RangeID/2/{u""-v""} | 01698a7572667468 | | | /Local/RangeID/2/u/RaftHardState | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000001 | | | /Local/RangeID/2/u/RaftLog/logIndex:1 | | /Local/RangeID/2/{u""-v""} | 01698a757266746c0000000000000002 | | | /Local/RangeID/2/u/RaftLog/logIndex:2 | | /Local/RangeID/2/{u""-v""} | 01698a75726c7274 | | | /Local/RangeID/2/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/2/{u""-v""} | 01698a753a61 | 01698a753a78 | 000000000000000109 | /Local/RangeID/2/u":{a"-x"}/0.000000001,0 | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output index c338a02b8446..256b75a810d3 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{b"-c"} | 016b1262ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"b\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{b"-c"} | 017a6b12016b126200ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"b"/RangeDescriptor | | /Local/Lock"{b"-c"} | 017a6b12620001 | | 030ce61c175eb445878c36dcf4062ada4c | "b" | +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output index 83056687a5df..3d0d6f5a9f46 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/2/{r""-s""} | 01698a726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/2/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/2/{r""-s""} | 01698a726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/2/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/2/{r""-s""} | 01698a726c67632d | | | /Local/RangeID/2/r/RangeGCThreshold | | /Local/RangeID/2/{r""-s""} | 01698a727261736b | | | /Local/RangeID/2/r/RangeAppliedState | | /Local/RangeID/2/{r""-s""} | 01698a72726c6c2d | | | /Local/RangeID/2/r/RangeLease | -| /Local/RangeID/2/{r""-s""} | 01698a723a61 | 01698a723a78 | 000000000000000109 | /Local/RangeID/2/r":{a"-x"}/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000172647363 | | 0000000000000001 | /Local/Range"b"/RangeDescriptor/0.000000001,0 | | /Local/Range"{b"-c"} | 016b1262000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"b"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{b"-c"} | 016b126200ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"b\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output index c70d703437d1..d66434a8982c 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r2/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | | {b-c} | 62 | | 0000000000000001 | "b"/0.000000001,0 | | {b-c} | 62ffffffff | | 0000000000000001 | "b\xff\xff\xff\xff"/0.000000001,0 | -| {b-c} | 62 | 63 | 000000000000000109 | {b-c}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output index a143fa468e13..7226bcc65239 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/all/output @@ -3,25 +3,25 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{c"-d"} | 016b1263ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"c\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{c"-d"} | 017a6b12016b126300ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"c"/RangeDescriptor | | /Local/Lock"{c"-d"} | 017a6b12630001 | | 030ce61c175eb445878c36dcf4062ada4c | "c" | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output index 396aafe31e65..4994b21a3cd8 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/exclude-user/output @@ -3,18 +3,18 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output index e146d6ec8b87..aad38dbbcb5e 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/all/user-only/output @@ -3,19 +3,19 @@ echo +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | +| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{u""-v""} | 01698b7572667462 | | | /Local/RangeID/3/u/RangeTombstone | | /Local/RangeID/3/{u""-v""} | 01698b7572667468 | | | /Local/RangeID/3/u/RaftHardState | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000001 | | | /Local/RangeID/3/u/RaftLog/logIndex:1 | | /Local/RangeID/3/{u""-v""} | 01698b757266746c0000000000000002 | | | /Local/RangeID/3/u/RaftLog/logIndex:2 | | /Local/RangeID/3/{u""-v""} | 01698b75726c7274 | | | /Local/RangeID/3/u/RangeLastReplicaGCTimestamp | -| /Local/RangeID/3/{u""-v""} | 01698b753a61 | 01698b753a78 | 000000000000000109 | /Local/RangeID/3/u":{a"-x"}/0.000000001,0 | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +----------------------------+--------------------------------------------------------+--------------+--------------------+---------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output index a0a0b86d22b4..53a85f4757a8 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/all/output @@ -3,19 +3,19 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/Range"{c"-d"} | 016b1263ffffffff000174786e2d295e727c8ca9437cbb5e8e2ebbad996f | | | /Local/Range"c\xff\xff\xff\xff"/Transaction/"295e727c-8ca9-437c-bb5e-8e2ebbad996f" | | /Local/Lock/Local/Range"{c"-d"} | 017a6b12016b126300ff01726473630001 | | 030ce61c175eb445878c36dcf4062ada4c | /Local/Range"c"/RangeDescriptor | | /Local/Lock"{c"-d"} | 017a6b12630001 | | 030ce61c175eb445878c36dcf4062ada4c | "c" | +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output index ac831e72ded7..22b8ec31e25b 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/exclude-user/output @@ -3,12 +3,12 @@ echo +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +---------------------------------+--------------------------------------------------------------+--------------+------------------------------------+------------------------------------------------------------------------------------+ +| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/RangeID/3/{r""-s""} | 01698b726162632d120ce61c175eb445878c36dcf4062ada4c0001 | | | /Local/RangeID/3/r/AbortSpan/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/RangeID/3/{r""-s""} | 01698b726162632d129855a1ef8eb94c06a106cab1dda78a2b0001 | | | /Local/RangeID/3/r/AbortSpan/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | | /Local/RangeID/3/{r""-s""} | 01698b726c67632d | | | /Local/RangeID/3/r/RangeGCThreshold | | /Local/RangeID/3/{r""-s""} | 01698b727261736b | | | /Local/RangeID/3/r/RangeAppliedState | | /Local/RangeID/3/{r""-s""} | 01698b72726c6c2d | | | /Local/RangeID/3/r/RangeLease | -| /Local/RangeID/3/{r""-s""} | 01698b723a61 | 01698b723a78 | 000000000000000109 | /Local/RangeID/3/r":{a"-x"}/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000172647363 | | 0000000000000001 | /Local/Range"c"/RangeDescriptor/0.000000001,0 | | /Local/Range"{c"-d"} | 016b1263000174786e2d0ce61c175eb445878c36dcf4062ada4c | | | /Local/Range"c"/Transaction/"0ce61c17-5eb4-4587-8c36-dcf4062ada4c" | | /Local/Range"{c"-d"} | 016b126300ff000174786e2d9855a1ef8eb94c06a106cab1dda78a2b | | | /Local/Range"c\x00"/Transaction/"9855a1ef-8eb9-4c06-a106-cab1dda78a2b" | diff --git a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output index 9949d61859e3..55c42ef5723e 100644 --- a/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output +++ b/pkg/kv/kvserver/rditer/testdata/TestReplicaDataIterator/r3/replicatedOnly/user-only/output @@ -3,7 +3,7 @@ echo +-------+------------+------------+--------------------+-----------------------------------+ | SPAN | KEY HEX | ENDKEY HEX | VERSION HEX | PRETTY | +-------+------------+------------+--------------------+-----------------------------------+ +| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | | {c-d} | 63 | | 0000000000000001 | "c"/0.000000001,0 | | {c-d} | 63ffffffff | | 0000000000000001 | "c\xff\xff\xff\xff"/0.000000001,0 | -| {c-d} | 63 | 64 | 000000000000000109 | {c-d}/0.000000001,0 | +-------+------------+------------+--------------------+-----------------------------------+ diff --git a/pkg/kv/kvserver/replica_command.go b/pkg/kv/kvserver/replica_command.go index 9ee3a984bd32..3926465fe0ce 100644 --- a/pkg/kv/kvserver/replica_command.go +++ b/pkg/kv/kvserver/replica_command.go @@ -3334,6 +3334,7 @@ func (r *Replica) followerSendSnapshot( SenderQueuePriority: req.SenderQueuePriority, SharedReplicate: sharedReplicate, ExternalReplicate: externalReplicate, + RangeKeysInOrder: true, } newBatchFn := func() storage.WriteBatch { return r.store.TODOEngine().NewWriteBatch() diff --git a/pkg/kv/kvserver/replica_learner_test.go b/pkg/kv/kvserver/replica_learner_test.go index 5986151a9af7..c0b848d46d54 100644 --- a/pkg/kv/kvserver/replica_learner_test.go +++ b/pkg/kv/kvserver/replica_learner_test.go @@ -2234,11 +2234,12 @@ func getExpectedSnapshotSizeBytes( err = rditer.IterateReplicaKeySpans( ctx, snap.State.Desc, snap.EngineSnap, true /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - switch keyType { - case storage.IterKeyTypePointsOnly: + hasPoint, hasRange := iter.HasPointAndRange() + + if hasPoint { unsafeKey, err := iter.UnsafeEngineKey() if err != nil { return err @@ -2250,8 +2251,8 @@ func getExpectedSnapshotSizeBytes( if err := b.PutEngineKey(unsafeKey, v); err != nil { return err } - - case storage.IterKeyTypeRangesOnly: + } + if hasRange && iter.RangeKeyChanged() { bounds, err := iter.EngineRangeBounds() if err != nil { return err @@ -2262,9 +2263,6 @@ func getExpectedSnapshotSizeBytes( return err } } - - default: - return errors.Errorf("unexpected key type %v", keyType) } } return err diff --git a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go index 6e13695a7e98..3757096942a6 100644 --- a/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go +++ b/pkg/kv/kvserver/replica_sst_snapshot_storage_test.go @@ -12,6 +12,7 @@ package kvserver import ( "context" + "encoding/binary" "fmt" io "io" "path/filepath" @@ -28,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" "github.com/cockroachdb/errors/oserror" @@ -283,7 +285,7 @@ func TestMultiSSTWriterInitSST(t *testing.T) { msstw, err := newMultiSSTWriter( ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, - false, /* skipRangeDelForMVCCSpan */ + false /* skipRangeDelForMVCCSpan */, false, /* rangeKeysInOrder */ ) require.NoError(t, err) _, err = msstw.Finish(ctx) @@ -365,16 +367,28 @@ func TestMultiSSTWriterSize(t *testing.T) { mvccSpan := keySpans[len(keySpans)-1] // Make a reference msstw with the default size. - referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false) + referenceMsstw, err := newMultiSSTWriter(ctx, settings, ref, localSpans, mvccSpan, 0, false, true /* rangeKeysInOrder */) require.NoError(t, err) require.Equal(t, int64(0), referenceMsstw.dataSize) + now := timeutil.Now().UnixNano() for i := range localSpans { require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) } - for i := 0; i < 100; i++ { - require.NoError(t, referenceMsstw.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + for i := 0; i < 1000; i++ { + key := binary.BigEndian.AppendUint32(append([]byte(nil), desc.StartKey...), uint32(i)) + mvccKey := storage.MVCCKey{Key: roachpb.Key(key), Timestamp: hlc.Timestamp{WallTime: now}} + engineKey, ok := storage.DecodeEngineKey(storage.EncodeMVCCKey(mvccKey)) + require.True(t, ok) + + if i%50 == 0 { + // Add a range key. + endKey := binary.BigEndian.AppendUint32(desc.StartKey, uint32(i+10)) + require.NoError(t, referenceMsstw.PutRangeKey( + ctx, key, endKey, storage.EncodeMVCCTimestampSuffix(mvccKey.Timestamp.WallPrev()), []byte(""))) + } + require.NoError(t, referenceMsstw.Put(ctx, engineKey, []byte("foobarbaz"))) } _, err = referenceMsstw.Finish(ctx) require.NoError(t, err) @@ -385,7 +399,7 @@ func TestMultiSSTWriterSize(t *testing.T) { MaxSnapshotSSTableSize.Override(ctx, &settings.SV, 100) - multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false) + multiSSTWriter, err := newMultiSSTWriter(ctx, settings, scratch, localSpans, mvccSpan, 0, false, true /* rangeKeysInOrder */) require.NoError(t, err) require.Equal(t, int64(0), multiSSTWriter.dataSize) @@ -393,8 +407,18 @@ func TestMultiSSTWriterSize(t *testing.T) { require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: localSpans[i].Key}, []byte("foo"))) } - for i := 0; i < 100; i++ { - require.NoError(t, multiSSTWriter.Put(ctx, storage.EngineKey{Key: roachpb.Key(append(desc.StartKey, byte(i)))}, []byte("foobarbaz"))) + for i := 0; i < 1000; i++ { + key := binary.BigEndian.AppendUint32(append([]byte(nil), desc.StartKey...), uint32(i)) + mvccKey := storage.MVCCKey{Key: roachpb.Key(key), Timestamp: hlc.Timestamp{WallTime: now}} + engineKey, ok := storage.DecodeEngineKey(storage.EncodeMVCCKey(mvccKey)) + require.True(t, ok) + if i%50 == 0 { + // Add a range key. + endKey := binary.BigEndian.AppendUint32(desc.StartKey, uint32(i+10)) + require.NoError(t, multiSSTWriter.PutRangeKey( + ctx, key, endKey, storage.EncodeMVCCTimestampSuffix(mvccKey.Timestamp.WallPrev()), []byte(""))) + } + require.NoError(t, multiSSTWriter.Put(ctx, engineKey, []byte("foobarbaz"))) } _, err = multiSSTWriter.Finish(ctx) @@ -462,7 +486,7 @@ func TestMultiSSTWriterAddLastSpan(t *testing.T) { msstw, err := newMultiSSTWriter( ctx, cluster.MakeTestingClusterSettings(), scratch, localSpans, mvccSpan, 0, - true, /* skipRangeDelForMVCCSpan */ + true /* skipRangeDelForMVCCSpan */, false, /* rangeKeysInOrder */ ) require.NoError(t, err) if addRangeDel { diff --git a/pkg/kv/kvserver/replica_test.go b/pkg/kv/kvserver/replica_test.go index de9f78c82c47..2b8a6277bf0f 100644 --- a/pkg/kv/kvserver/replica_test.go +++ b/pkg/kv/kvserver/replica_test.go @@ -7198,8 +7198,7 @@ func TestReplicaDestroy(t *testing.T) { require.NoError(t, rditer.IterateReplicaKeySpans( ctx, tc.repl.Desc(), engSnapshot, false /* replicatedOnly */, rditer.ReplicatedSpansAll, - func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { - require.Equal(t, storage.IterKeyTypePointsOnly, keyType) + func(iter storage.EngineIterator, _ roachpb.Span) error { var err error for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { key, err := iter.UnsafeEngineKey() diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 01a34a952033..d1bd4156012c 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -35,6 +35,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/metamorphic" "github.com/cockroachdb/cockroach/pkg/util/metric" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -70,8 +71,13 @@ var MaxSnapshotSSTableSize = settings.RegisterByteSizeSetting( settings.SystemOnly, "kv.snapshot_rebalance.max_sst_size", "maximum size of a rebalance or recovery SST size", - 128<<20, // 128 MB - settings.PositiveInt, + int64(metamorphic.ConstantWithTestRange( + "kv.snapshot_rebalance.max_sst_size", + 128<<20, /* defaultValue */ + 32<<10, /* metamorphic min */ + 512<<20, /* metamorphic max */ + )), // 128 MB default + settings.NonNegativeInt, ) // snapshotMetrics contains metrics on the number and size of snapshots in @@ -175,6 +181,7 @@ func newMultiSSTWriter( mvccKeySpan roachpb.Span, sstChunkSize int64, skipClearForMVCCSpan bool, + rangeKeysInOrder bool, ) (*multiSSTWriter, error) { msstw := &multiSSTWriter{ st: st, @@ -188,10 +195,17 @@ func newMultiSSTWriter( sstChunkSize: sstChunkSize, skipClearForMVCCSpan: skipClearForMVCCSpan, } - if !skipClearForMVCCSpan { + if !skipClearForMVCCSpan && rangeKeysInOrder { // If skipClearForMVCCSpan is true, we don't split the MVCC span across // multiple sstables, as addClearForMVCCSpan could be called by the caller // at any time. + // + // We also disable snapshot sstable splitting unless the sender has + // specified in its snapshot header that it is sending range keys in + // key order alongside point keys, as opposed to sending them at the end + // of the snapshot. This is necessary to efficiently produce fragmented + // snapshot sstables, as otherwise range keys will arrive out-of-order + // wrt. point keys. msstw.maxSSTSize = MaxSnapshotSSTableSize.Get(&st.SV) } msstw.rangeKeyFrag = rangekey.Fragmenter{ @@ -697,7 +711,7 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( // The last key range is the user key span. localRanges := keyRanges[:len(keyRanges)-1] mvccRange := keyRanges[len(keyRanges)-1] - msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan) + msstw, err := newMultiSSTWriter(ctx, kvSS.st, kvSS.scratch, localRanges, mvccRange, kvSS.sstChunkSize, skipClearForMVCCSpan, header.RangeKeysInOrder) if err != nil { return noSnap, err } @@ -987,14 +1001,33 @@ func (kvSS *kvBatchSnapshotStrategy) Send( replicatedFilter = rditer.ReplicatedSpansExcludeUser } - iterateRKSpansVisitor := func(iter storage.EngineIterator, _ roachpb.Span, keyType storage.IterKeyType) error { + iterateRKSpansVisitor := func(iter storage.EngineIterator, _ roachpb.Span) error { timingTag.start("iter") defer timingTag.stop("iter") var err error - switch keyType { - case storage.IterKeyTypePointsOnly: - for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { + for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { + hasPoint, hasRange := iter.HasPointAndRange() + if hasRange && iter.RangeKeyChanged() { + bounds, err := iter.EngineRangeBounds() + if err != nil { + return err + } + for _, rkv := range iter.EngineRangeKeys() { + rangeKVs++ + if b == nil { + b = kvSS.newWriteBatch() + } + err := b.PutEngineRangeKey(bounds.Key, bounds.EndKey, rkv.Version, rkv.Value) + if err != nil { + return err + } + if err = maybeFlushBatch(); err != nil { + return err + } + } + } + if hasPoint { kvs++ if b == nil { b = kvSS.newWriteBatch() @@ -1014,30 +1047,6 @@ func (kvSS *kvBatchSnapshotStrategy) Send( return err } } - - case storage.IterKeyTypeRangesOnly: - for ok := true; ok && err == nil; ok, err = iter.NextEngineKey() { - bounds, err := iter.EngineRangeBounds() - if err != nil { - return err - } - for _, rkv := range iter.EngineRangeKeys() { - rangeKVs++ - if b == nil { - b = kvSS.newWriteBatch() - } - err := b.PutEngineRangeKey(bounds.Key, bounds.EndKey, rkv.Version, rkv.Value) - if err != nil { - return err - } - if err = maybeFlushBatch(); err != nil { - return err - } - } - } - - default: - return errors.AssertionFailedf("unexpected key type %v", keyType) } return err } @@ -1961,6 +1970,7 @@ func SendEmptySnapshot( State: state, RaftMessageRequest: req, RangeSize: ms.Total(), + RangeKeysInOrder: true, } stream, err := NewMultiRaftClient(cc).RaftSnapshot(ctx)