From aec78a5feb678c66be85e786f947ed50ee6264eb Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Tue, 27 Aug 2024 00:21:42 +0530 Subject: [PATCH 1/4] passing zap specific config via segPlugin interface --- go.mod | 12 ++++++++++++ go.sum | 12 ------------ index/scorch/merge.go | 4 ++-- index/scorch/persister.go | 4 ++-- index/scorch/scorch.go | 2 +- index/scorch/segment_plugin.go | 4 ++++ 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index de12890f5..388af2484 100644 --- a/go.mod +++ b/go.mod @@ -43,3 +43,15 @@ require ( github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.13.0 // indirect ) + +replace github.com/blevesearch/zapx/v16 => /Users/thejas.orkombu/fts/blevesearch/zapx + +replace github.com/blevesearch/zapx/v15 => /Users/thejas.orkombu/fts/blevesearch/zapx15 + +replace github.com/blevesearch/zapx/v14 => /Users/thejas.orkombu/fts/blevesearch/zapx14 + +replace github.com/blevesearch/zapx/v13 => /Users/thejas.orkombu/fts/blevesearch/zapx13 + +replace github.com/blevesearch/zapx/v12 => /Users/thejas.orkombu/fts/blevesearch/zapx12 + +replace github.com/blevesearch/zapx/v11 => /Users/thejas.orkombu/fts/blevesearch/zapx11 diff --git a/go.sum b/go.sum index 51d0ece96..606052de1 100644 --- a/go.sum +++ b/go.sum @@ -33,18 +33,6 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= -github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= -github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= -github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= -github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= -github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= -github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= -github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= -github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.1.6-0.20240805195258-e1776480442c h1:j2znHQQ7LfLtwPNRekgv49D0IbrLkmIMzKAhho3X0L0= -github.com/blevesearch/zapx/v16 v16.1.6-0.20240805195258-e1776480442c/go.mod h1:Er6ZhsETdPDgHUm7EPlgaNsDz/PuX9fokakN3ZrBYsQ= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/merge.go b/index/scorch/merge.go index b74504ca1..23ade5a70 100644 --- a/index/scorch/merge.go +++ b/index/scorch/merge.go @@ -358,7 +358,7 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context, return fmt.Errorf("merging failed: %v", err) } - seg, err = s.segPlugin.Open(path) + seg, err = s.segPlugin.OpenEx(path, s.config) if err != nil { s.unmarkIneligibleForRemoval(filename) atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) @@ -484,7 +484,7 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, return nil, 0, err } - seg, err := s.segPlugin.Open(path) + seg, err := s.segPlugin.OpenEx(path, s.config) if err != nil { atomic.AddUint64(&s.stats.TotMemMergeErr, 1) return nil, 0, err diff --git a/index/scorch/persister.go b/index/scorch/persister.go index d59f733df..bf9227589 100644 --- a/index/scorch/persister.go +++ b/index/scorch/persister.go @@ -662,7 +662,7 @@ func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { } }() for segmentID, path := range newSegmentPaths { - newSegments[segmentID], err = s.segPlugin.Open(path) + newSegments[segmentID], err = s.segPlugin.OpenEx(path, s.config) if err != nil { return fmt.Errorf("error opening new segment at %s, %v", path, err) } @@ -872,7 +872,7 @@ func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, erro return nil, fmt.Errorf("segment path missing") } segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) - segment, err := s.segPlugin.Open(segmentPath) + segment, err := s.segPlugin.OpenEx(segmentPath, s.config) if err != nil { return nil, fmt.Errorf("error opening bolt segment: %v", err) } diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index 429d1daa9..b90a29f28 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -441,7 +441,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { stats := newFieldStats() if len(analysisResults) > 0 { - newSegment, bufBytes, err = s.segPlugin.New(analysisResults) + newSegment, bufBytes, err = s.segPlugin.NewEx(analysisResults, s.config) if err != nil { return err } diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index b3b9ba01f..f7772db2a 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -46,10 +46,14 @@ type SegmentPlugin interface { // New takes a set of Documents and turns them into a new Segment New(results []index.Document) (segment.Segment, uint64, error) + NewEx(results []index.Document, config map[string]interface{}) (segment.Segment, uint64, error) + // Open attempts to open the file at the specified path and // return the corresponding Segment Open(path string) (segment.Segment, error) + OpenEx(path string, config map[string]interface{}) (segment.Segment, error) + // Merge takes a set of Segments, and creates a new segment on disk at // the specified path. // Drops is a set of bitmaps (one for each segment) indicating which From bdd497abf9a24a4cc0fd9b8ccb58eaf0d7e38c99 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Tue, 27 Aug 2024 20:35:51 +0530 Subject: [PATCH 2/4] segment config member variable --- index/scorch/merge.go | 4 ++-- index/scorch/persister.go | 4 ++-- index/scorch/scorch.go | 11 ++++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/index/scorch/merge.go b/index/scorch/merge.go index 23ade5a70..280c2dbec 100644 --- a/index/scorch/merge.go +++ b/index/scorch/merge.go @@ -358,7 +358,7 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context, return fmt.Errorf("merging failed: %v", err) } - seg, err = s.segPlugin.OpenEx(path, s.config) + seg, err = s.segPlugin.OpenEx(path, s.segmentConfig) if err != nil { s.unmarkIneligibleForRemoval(filename) atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) @@ -484,7 +484,7 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, return nil, 0, err } - seg, err := s.segPlugin.OpenEx(path, s.config) + seg, err := s.segPlugin.OpenEx(path, s.segmentConfig) if err != nil { atomic.AddUint64(&s.stats.TotMemMergeErr, 1) return nil, 0, err diff --git a/index/scorch/persister.go b/index/scorch/persister.go index bf9227589..6e272e1ad 100644 --- a/index/scorch/persister.go +++ b/index/scorch/persister.go @@ -662,7 +662,7 @@ func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { } }() for segmentID, path := range newSegmentPaths { - newSegments[segmentID], err = s.segPlugin.OpenEx(path, s.config) + newSegments[segmentID], err = s.segPlugin.OpenEx(path, s.segmentConfig) if err != nil { return fmt.Errorf("error opening new segment at %s, %v", path, err) } @@ -872,7 +872,7 @@ func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, erro return nil, fmt.Errorf("segment path missing") } segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) - segment, err := s.segPlugin.OpenEx(segmentPath, s.config) + segment, err := s.segPlugin.OpenEx(segmentPath, s.segmentConfig) if err != nil { return nil, fmt.Errorf("error opening bolt segment: %v", err) } diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index b90a29f28..219da07af 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -44,6 +44,7 @@ type Scorch struct { readOnly bool version uint8 config map[string]interface{} + segmentConfig map[string]interface{} analysisQueue *index.AnalysisQueue path string @@ -323,6 +324,14 @@ func (s *Scorch) openBolt() error { s.rollbackRetentionFactor = r } + if v, ok := s.config["cacheExpiryTime"]; ok { + // validate right at the beginning + if _, err = parseToTimeDuration(v); err != nil { + return fmt.Errorf("cacheExpiryTime parse err: %v", err) + } + s.segmentConfig["cacheExpiryTime"] = v + } + typ, ok := s.config["spatialPlugin"].(string) if ok { s.loadSpatialAnalyzerPlugin(typ) @@ -441,7 +450,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { stats := newFieldStats() if len(analysisResults) > 0 { - newSegment, bufBytes, err = s.segPlugin.NewEx(analysisResults, s.config) + newSegment, bufBytes, err = s.segPlugin.NewEx(analysisResults, s.segmentConfig) if err != nil { return err } From c49bc71b4b56feafcdf6c2db86757ab997d1bc17 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Thu, 29 Aug 2024 12:27:52 +0530 Subject: [PATCH 3/4] segmentConfig init --- go.mod | 12 ------------ go.sum | 12 ++++++++++++ index/scorch/scorch.go | 9 +-------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 388af2484..de12890f5 100644 --- a/go.mod +++ b/go.mod @@ -43,15 +43,3 @@ require ( github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.13.0 // indirect ) - -replace github.com/blevesearch/zapx/v16 => /Users/thejas.orkombu/fts/blevesearch/zapx - -replace github.com/blevesearch/zapx/v15 => /Users/thejas.orkombu/fts/blevesearch/zapx15 - -replace github.com/blevesearch/zapx/v14 => /Users/thejas.orkombu/fts/blevesearch/zapx14 - -replace github.com/blevesearch/zapx/v13 => /Users/thejas.orkombu/fts/blevesearch/zapx13 - -replace github.com/blevesearch/zapx/v12 => /Users/thejas.orkombu/fts/blevesearch/zapx12 - -replace github.com/blevesearch/zapx/v11 => /Users/thejas.orkombu/fts/blevesearch/zapx11 diff --git a/go.sum b/go.sum index 606052de1..51d0ece96 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,18 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v16 v16.1.6-0.20240805195258-e1776480442c h1:j2znHQQ7LfLtwPNRekgv49D0IbrLkmIMzKAhho3X0L0= +github.com/blevesearch/zapx/v16 v16.1.6-0.20240805195258-e1776480442c/go.mod h1:Er6ZhsETdPDgHUm7EPlgaNsDz/PuX9fokakN3ZrBYsQ= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index 219da07af..5901ac34e 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -122,6 +122,7 @@ func NewScorch(storeName string, forceMergeRequestCh: make(chan *mergerCtrl, 1), segPlugin: defaultSegmentPlugin, copyScheduled: map[string]int{}, + segmentConfig: make(map[string]interface{}), } forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config) @@ -324,14 +325,6 @@ func (s *Scorch) openBolt() error { s.rollbackRetentionFactor = r } - if v, ok := s.config["cacheExpiryTime"]; ok { - // validate right at the beginning - if _, err = parseToTimeDuration(v); err != nil { - return fmt.Errorf("cacheExpiryTime parse err: %v", err) - } - s.segmentConfig["cacheExpiryTime"] = v - } - typ, ok := s.config["spatialPlugin"].(string) if ok { s.loadSpatialAnalyzerPlugin(typ) From de21e7277f9e3e03327049700d5d23cedb3dae97 Mon Sep 17 00:00:00 2001 From: Thejas-bhat Date: Fri, 6 Sep 2024 11:15:34 +0530 Subject: [PATCH 4/4] passing config for Merge() as well --- index/scorch/merge.go | 6 +++--- index/scorch/segment_plugin.go | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/index/scorch/merge.go b/index/scorch/merge.go index 280c2dbec..663c258cd 100644 --- a/index/scorch/merge.go +++ b/index/scorch/merge.go @@ -339,8 +339,8 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context, atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) prevBytesReadTotal := cumulateBytesRead(segmentsToMerge) - newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path, - cw.cancelCh, s) + newDocNums, _, err := s.segPlugin.MergeEx(segmentsToMerge, docsToDrop, path, + cw.cancelCh, s, s.segmentConfig) atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) @@ -469,7 +469,7 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, path := s.path + string(os.PathSeparator) + filename newDocNums, _, err := - s.segPlugin.Merge(sbs, sbsDrops, path, s.closeCh, s) + s.segPlugin.MergeEx(sbs, sbsDrops, path, s.closeCh, s, s.segmentConfig) atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1) diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go index f7772db2a..cf12cf928 100644 --- a/index/scorch/segment_plugin.go +++ b/index/scorch/segment_plugin.go @@ -71,6 +71,10 @@ type SegmentPlugin interface { Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, closeCh chan struct{}, s segment.StatsReporter) ( [][]uint64, uint64, error) + + MergeEx(segments []segment.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s segment.StatsReporter, config map[string]interface{}) ( + [][]uint64, uint64, error) } var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin