Adding verbose output option to server and fixing pebble bulk insert …

…compaction rate
bmeg · Feb 7, 2024 · 37b04e3 · 37b04e3
1 parent ddf6dee
commit 37b04e3
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 10 deletions.
diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -18,6 +18,7 @@ import (
 var conf = &config.Config{}
 var configFile string
 var driver = "badger"
+var verbose bool
 
 var endPoints = map[string]string{}
 
@@ -102,6 +103,11 @@ var Cmd = &cobra.Command{
 				conf.RPCClient.ServerAddress = conf.Server.RPCAddress()
 			}
 		}
+
+		if verbose {
+			conf.Logger.Level = "debug"
+		}
+
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -119,6 +125,8 @@ func init() {
 	flags.StringVar(&conf.Logger.Formatter, "log-format", conf.Logger.Formatter, "Log format [text, json]")
 	flags.BoolVar(&conf.Server.RequestLogging.Enable, "log-requests", conf.Server.RequestLogging.Enable, "Log all requests")
 
+	flags.BoolVar(&verbose, "verbose", verbose, "Verbose")
+
 	flags.StringVarP(&pluginDir, "plugins", "p", pluginDir, "Directory with GRIPPER plugins")
 	flags.StringVarP(&driver, "driver", "d", driver, "Default Driver")
 

diff --git a/gdbi/jsonpath.go b/gdbi/jsonpath.go
@@ -48,9 +48,17 @@ import (
 func GetDoc(traveler Traveler, namespace string) map[string]interface{} {
 	var tmap map[string]interface{}
 	if namespace == travelerpath.Current {
-		tmap = traveler.GetCurrent().Get().ToDict()
+		dr := traveler.GetCurrent()
+		if dr == nil {
+			return nil
+		}
+		tmap = dr.Get().ToDict()
 	} else {
-		tmap = traveler.GetMark(namespace).Get().ToDict()
+		dr := traveler.GetMark(namespace)
+		if dr == nil {
+			return nil
+		}
+		tmap = dr.Get().ToDict()
 	}
 	return tmap
 }

diff --git a/gdbi/traveler.go b/gdbi/traveler.go
@@ -38,13 +38,15 @@ func (t *BaseTraveler) AddCurrent(r DataRef) Traveler {
 	for i := range t.Path {
 		o.Path[i] = t.Path[i]
 	}
-	rd := r.Get()
-	if rd == nil {
-		o.Path[len(t.Path)] = DataElementID{}
-	} else if rd.To != "" {
-		o.Path[len(t.Path)] = DataElementID{Edge: rd.ID}
-	} else {
-		o.Path[len(t.Path)] = DataElementID{Vertex: rd.ID}
+	if r != nil {
+		rd := r.Get()
+		if rd == nil {
+			o.Path[len(t.Path)] = DataElementID{}
+		} else if rd.To != "" {
+			o.Path[len(t.Path)] = DataElementID{Edge: rd.ID}
+		} else {
+			o.Path[len(t.Path)] = DataElementID{Vertex: rd.ID}
+		}
 	}
 	o.Current = r
 	return &o

diff --git a/kvi/pebbledb/pebble_store.go b/kvi/pebbledb/pebble_store.go
@@ -19,6 +19,8 @@ import (
 
 var loaded = kvi.AddKVDriver("pebble", NewKVInterface)
 
+var defaultCompactLimit = uint32(10000)
+
 // PebbleKV is an implementation of the KVStore for badger
 type PebbleKV struct {
 	db           *pebble.DB
@@ -32,7 +34,7 @@ func NewKVInterface(path string, kopts kvi.Options) (kvi.KVInterface, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &PebbleKV{db: db}, nil
+	return &PebbleKV{db: db, insertCount: 0, compactLimit: defaultCompactLimit}, nil
 }
 
 func WrapPebble(db *pebble.DB) kvi.KVInterface {
@@ -269,6 +271,7 @@ func (pdb *PebbleKV) BulkWrite(u func(tx kvi.KVBulkWrite) error) error {
 
 	pdb.insertCount += ptx.totalInserts
 	if pdb.insertCount > pdb.compactLimit {
+		log.Debugf("Running pebble compact %d > %d", pdb.insertCount, pdb.compactLimit)
 		//pdb.db.Compact(ptx.lowest, ptx.highest, true)
 		pdb.db.Compact([]byte{0x00}, []byte{0xFF}, true)
 		pdb.insertCount = 0