Traveler Refactor

benchmark/kv-query-profile/main.go

@@ -8,7 +8,6 @@ import (
 	"log"
 	"os"
 	"runtime/pprof"
-	"strings"
 	"time"
 
 	"github.com/bmeg/grip/engine/pipeline"
@@ -17,7 +16,7 @@ import (
 	"github.com/bmeg/grip/kvgraph"
 	"github.com/bmeg/grip/kvi"
 	"github.com/dop251/goja"
-	"github.com/golang/protobuf/jsonpb"
+	"google.golang.org/protobuf/encoding/protojson"
 
 	gripqljs "github.com/bmeg/grip/gripql/javascript"
 
@@ -76,7 +75,7 @@ func main() {
 	}
 
 	query := gripql.GraphQuery{}
-	err = jsonpb.Unmarshal(strings.NewReader(string(queryJSON)), &query)
+	err = protojson.Unmarshal(queryJSON, &query)
 	if err != nil {
 		log.Printf("%s", err)
 		return

cmd/job/main.go

@@ -1,17 +1,14 @@
 package job
 
-
 import (
 	"fmt"
-	"encoding/json"
+
 	"github.com/bmeg/grip/gripql"
+	gripqljs "github.com/bmeg/grip/gripql/javascript"
+	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	"github.com/bmeg/grip/util/rpc"
 	"github.com/spf13/cobra"
-	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	"google.golang.org/protobuf/encoding/protojson"
-	"github.com/dop251/goja"
-	gripqljs "github.com/bmeg/grip/gripql/javascript"
-	"github.com/bmeg/grip/jsengine/underscore"
 )
 
 var host = "localhost:8202"
@@ -28,7 +25,7 @@ var listJobsCmd = &cobra.Command{
 	Long:  ``,
 	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
+		graph := args[0]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -48,13 +45,13 @@ var listJobsCmd = &cobra.Command{
 }
 
 var dropCmd = &cobra.Command{
-	Use:   "drop",
-	Short: "List graphs",
+	Use:   "drop <job>",
+	Short: "Drop job",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
-    jobID := args[1]
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -65,19 +62,19 @@ var dropCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-    fmt.Printf("%s\n", resp)
+		fmt.Printf("%s\n", resp)
 		return nil
 	},
 }
 
 var getCmd = &cobra.Command{
-	Use:   "get job",
+	Use:   "get <job>",
 	Short: "Get job info",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
-    jobID := args[1]
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -103,70 +100,110 @@ var getCmd = &cobra.Command{
 	},
 }
 
-
 var submitCmd = &cobra.Command{
 	Use:   "submit <graph> <query>",
 	Short: "Submit query job",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
+		graph := args[0]
 		queryString := args[1]
 
-		vm := goja.New()
-		us, err := underscore.Asset("underscore.js")
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
-			return fmt.Errorf("failed to load underscore.js")
-		}
-		if _, err := vm.RunString(string(us)); err != nil {
 			return err
 		}
-		gripqlString, err := gripqljs.Asset("gripql.js")
+		query.Graph = graph
+
+		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
-			return fmt.Errorf("failed to load gripql.js")
+			return err
 		}
-		if _, err := vm.RunString(string(gripqlString)); err != nil {
+
+		res, err := conn.Submit(query)
+		if err != nil {
 			return err
 		}
 
-		val, err := vm.RunString(queryString)
+		fmt.Printf("%s\n", res)
+		return nil
+	},
+}
+
+var resumeCmd = &cobra.Command{
+	Use:   "resume <graph> <job> <query>",
+	Short: "Resume query job",
+	Long:  ``,
+	Args:  cobra.ExactArgs(3),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		graph := args[0]
+		jobID := args[1]
+		queryString := args[2]
+
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
 			return err
 		}
+		query.Graph = graph
 
-		queryJSON, err := json.Marshal(val)
+		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
 		}
 
-		query := gripql.GraphQuery{}
-		err = protojson.Unmarshal(queryJSON, &query)
+		res, err := conn.ResumeJob(graph, jobID, query)
 		if err != nil {
 			return err
 		}
-		query.Graph = graph
+
+		for row := range res {
+			rowString, _ := protojson.Marshal(row)
+			fmt.Printf("%s\n", rowString)
+		}
+		return nil
+
+	},
+}
+
+var viewCmd = &cobra.Command{
+	Use:   "view <graph> <job>",
+	Short: "Resume query job",
+	Long:  ``,
+	Args:  cobra.ExactArgs(2),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
 		}
 
-		res, err := conn.Submit(&query)
+		res, err := conn.ViewJob(graph, jobID)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%s\n", res)
+		for row := range res {
+			rowString, _ := protojson.Marshal(row)
+			fmt.Printf("%s\n", rowString)
+		}
 		return nil
 	},
 }
 
 func init() {
 	listJobsCmd.Flags().StringVar(&host, "host", host, "grip server url")
 	getCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	viewCmd.Flags().StringVar(&host, "host", host, "grip server url")
 	dropCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	submitCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	resumeCmd.Flags().StringVar(&host, "host", host, "grip server url")
 
 	Cmd.AddCommand(listJobsCmd)
 	Cmd.AddCommand(getCmd)
+	Cmd.AddCommand(viewCmd)
 	Cmd.AddCommand(dropCmd)
+	Cmd.AddCommand(submitCmd)
+	Cmd.AddCommand(resumeCmd)
 }

cmd/query/main.go

@@ -1,16 +1,13 @@
 package query
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/bmeg/grip/gripql"
 	gripqljs "github.com/bmeg/grip/gripql/javascript"
 	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	_ "github.com/bmeg/grip/jsengine/otto" // import otto so it registers with the driver map
-	"github.com/bmeg/grip/jsengine/underscore"
 	"github.com/bmeg/grip/util/rpc"
-	"github.com/dop251/goja"
 	"github.com/spf13/cobra"
 	"google.golang.org/protobuf/encoding/protojson"
 )
@@ -26,48 +23,20 @@ Example:
     grip query example-graph 'V().hasLabel("Variant").out().limit(5)'`,
 	Args: cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		vm := goja.New()
-
-		us, err := underscore.Asset("underscore.js")
-		if err != nil {
-			return fmt.Errorf("failed to load underscore.js")
-		}
-		if _, err := vm.RunString(string(us)); err != nil {
-			return err
-		}
-
-		gripqlString, err := gripqljs.Asset("gripql.js")
-		if err != nil {
-			return fmt.Errorf("failed to load gripql.js")
-		}
-		if _, err := vm.RunString(string(gripqlString)); err != nil {
-			return err
-		}
-
+		graph := args[0]
 		queryString := args[1]
-		val, err := vm.RunString(queryString)
-		if err != nil {
-			return err
-		}
 
-		queryJSON, err := json.Marshal(val)
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
 			return err
 		}
-
-		query := gripql.GraphQuery{}
-		err = protojson.Unmarshal(queryJSON, &query)
-		if err != nil {
-			return err
-		}
-		query.Graph = args[0]
-
+		query.Graph = graph
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
 		}
 
-		res, err := conn.Traversal(&query)
+		res, err := conn.Traversal(query)
 		if err != nil {
 			return err
 		}

cmd/server/main.go

@@ -18,6 +18,7 @@ import (
 var conf = &config.Config{}
 var configFile string
 var driver = "badger"
+var verbose bool
 
 var endPoints = map[string]string{}
 
@@ -81,6 +82,8 @@ var Cmd = &cobra.Command{
 				dconf.AddPebbleDefault()
 			} else if driver == "mongo" {
 				dconf.AddMongoDefault()
+			} else if driver == "grids" {
+				dconf.AddGridsDefault()
 			}
 		}
 		if pluginDir != "" {
@@ -100,6 +103,11 @@ var Cmd = &cobra.Command{
 				conf.RPCClient.ServerAddress = conf.Server.RPCAddress()
 			}
 		}
+
+		if verbose {
+			conf.Logger.Level = "debug"
+		}
+
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -117,6 +125,8 @@ func init() {
 	flags.StringVar(&conf.Logger.Formatter, "log-format", conf.Logger.Formatter, "Log format [text, json]")
 	flags.BoolVar(&conf.Server.RequestLogging.Enable, "log-requests", conf.Server.RequestLogging.Enable, "Log all requests")
 
+	flags.BoolVar(&verbose, "verbose", verbose, "Verbose")
+
 	flags.StringVarP(&pluginDir, "plugins", "p", pluginDir, "Directory with GRIPPER plugins")
 	flags.StringVarP(&driver, "driver", "d", driver, "Default Driver")
 

config/config.go

@@ -101,6 +101,12 @@ func (conf *Config) AddMongoDefault() {
 	conf.Default = "mongo"
 }
 
+func (conf *Config) AddGridsDefault() {
+	n := "grip-grids.db"
+	conf.Drivers["grids"] = DriverConfig{Grids: &n}
+	conf.Default = "grids"
+}
+
 // TestifyConfig randomizes ports and database paths/names
 func TestifyConfig(c *Config) {
 	rand := strings.ToLower(util.RandomString(6))

conformance/tests/ot_aggregations.py

@@ -122,7 +122,7 @@ def getMinMax(input_data, percent, accuracy=0.15):
     if count != len(percents):
         errors.append(
             "Unexpected number of terms: %d != %d" %
-            (len(row["buckets"]), len(percents))
+            (len(res["buckets"]), len(percents))
         )
 
     return errors
@@ -190,18 +190,19 @@ def test_traversal_gid_aggregation(man):
 def test_field_aggregation(man):
     errors = []
 
-    fields = [ "id", 'orbital_period', 'gravity', 'terrain', 'name','climate', 'system', 'diameter', 'rotation_period', 'url', 'population', 'surface_water']
+    # TODO: find way to get gripper driver to drop id field
+    fields = [ "_id", "id", "_gid", "_label", 'orbital_period', 'gravity', 'terrain', 'name','climate', 'system', 'diameter', 'rotation_period', 'url', 'population', 'surface_water']
 
     G = man.setGraph("swapi")
     count = 0
-    for row in G.query().V().hasLabel("Planet").aggregate(gripql.field("gid-agg", "$._data")):
+    for row in G.query().V().hasLabel("Planet").aggregate(gripql.field("gid-agg", "$")):
         if row["key"] not in fields:
             errors.append("unknown field returned: %s" % (row['key']))
         if row["value"] != 3:
             errors.append("incorrect count returned: %s" % (row['value']))
         count += 1
-    if count not in [11, 12]: # gripper returns an id field as well, others dont....
-        errors.append("Incorrect number of results returned")
+    if count not in [11, 12, 13]: # gripper returns an id field as well, others dont....
+        errors.append("""V().hasLabel("Planet").aggregate(gripql.field("gid-agg", "$")) : Incorrect number of results returned %d""" % (count))
     return errors