Merge branch 'develop' into feature/grip-client-context

bmeg · Apr 11, 2024 · e350317 · e350317
2 parents 804b1a5 + 87975df
commit e350317
Show file tree

Hide file tree

Showing 104 changed files with 3,313 additions and 6,084 deletions.
diff --git a/benchmark/kv-query-profile/main.go b/benchmark/kv-query-profile/main.go
@@ -8,7 +8,6 @@ import (
 	"log"
 	"os"
 	"runtime/pprof"
-	"strings"
 	"time"
 
 	"github.com/bmeg/grip/engine/pipeline"
@@ -17,7 +16,7 @@ import (
 	"github.com/bmeg/grip/kvgraph"
 	"github.com/bmeg/grip/kvi"
 	"github.com/dop251/goja"
-	"github.com/golang/protobuf/jsonpb"
+	"google.golang.org/protobuf/encoding/protojson"
 
 	gripqljs "github.com/bmeg/grip/gripql/javascript"
 
@@ -76,7 +75,7 @@ func main() {
 	}
 
 	query := gripql.GraphQuery{}
-	err = jsonpb.Unmarshal(strings.NewReader(string(queryJSON)), &query)
+	err = protojson.Unmarshal(queryJSON, &query)
 	if err != nil {
 		log.Printf("%s", err)
 		return

diff --git a/cmd/job/main.go b/cmd/job/main.go
@@ -1,17 +1,14 @@
 package job
 
-
 import (
 	"fmt"
-	"encoding/json"
+
 	"github.com/bmeg/grip/gripql"
+	gripqljs "github.com/bmeg/grip/gripql/javascript"
+	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	"github.com/bmeg/grip/util/rpc"
 	"github.com/spf13/cobra"
-	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	"google.golang.org/protobuf/encoding/protojson"
-	"github.com/dop251/goja"
-	gripqljs "github.com/bmeg/grip/gripql/javascript"
-	"github.com/bmeg/grip/jsengine/underscore"
 )
 
 var host = "localhost:8202"
@@ -28,7 +25,7 @@ var listJobsCmd = &cobra.Command{
 	Long:  ``,
 	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
+		graph := args[0]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -48,13 +45,13 @@ var listJobsCmd = &cobra.Command{
 }
 
 var dropCmd = &cobra.Command{
-	Use:   "drop",
-	Short: "List graphs",
+	Use:   "drop <job>",
+	Short: "Drop job",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
-    jobID := args[1]
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -65,19 +62,19 @@ var dropCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-    fmt.Printf("%s\n", resp)
+		fmt.Printf("%s\n", resp)
 		return nil
 	},
 }
 
 var getCmd = &cobra.Command{
-	Use:   "get job",
+	Use:   "get <job>",
 	Short: "Get job info",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
-    jobID := args[1]
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
@@ -103,70 +100,110 @@ var getCmd = &cobra.Command{
 	},
 }
 
-
 var submitCmd = &cobra.Command{
 	Use:   "submit <graph> <query>",
 	Short: "Submit query job",
 	Long:  ``,
 	Args:  cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-    graph := args[0]
+		graph := args[0]
 		queryString := args[1]
 
-		vm := goja.New()
-		us, err := underscore.Asset("underscore.js")
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
-			return fmt.Errorf("failed to load underscore.js")
-		}
-		if _, err := vm.RunString(string(us)); err != nil {
 			return err
 		}
-		gripqlString, err := gripqljs.Asset("gripql.js")
+		query.Graph = graph
+
+		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
-			return fmt.Errorf("failed to load gripql.js")
+			return err
 		}
-		if _, err := vm.RunString(string(gripqlString)); err != nil {
+
+		res, err := conn.Submit(query)
+		if err != nil {
 			return err
 		}
 
-		val, err := vm.RunString(queryString)
+		fmt.Printf("%s\n", res)
+		return nil
+	},
+}
+
+var resumeCmd = &cobra.Command{
+	Use:   "resume <graph> <job> <query>",
+	Short: "Resume query job",
+	Long:  ``,
+	Args:  cobra.ExactArgs(3),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		graph := args[0]
+		jobID := args[1]
+		queryString := args[2]
+
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
 			return err
 		}
+		query.Graph = graph
 
-		queryJSON, err := json.Marshal(val)
+		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
 		}
 
-		query := gripql.GraphQuery{}
-		err = protojson.Unmarshal(queryJSON, &query)
+		res, err := conn.ResumeJob(graph, jobID, query)
 		if err != nil {
 			return err
 		}
-		query.Graph = graph
+
+		for row := range res {
+			rowString, _ := protojson.Marshal(row)
+			fmt.Printf("%s\n", rowString)
+		}
+		return nil
+
+	},
+}
+
+var viewCmd = &cobra.Command{
+	Use:   "view <graph> <job>",
+	Short: "Resume query job",
+	Long:  ``,
+	Args:  cobra.ExactArgs(2),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		graph := args[0]
+		jobID := args[1]
 
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
 		}
 
-		res, err := conn.Submit(&query)
+		res, err := conn.ViewJob(graph, jobID)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%s\n", res)
+		for row := range res {
+			rowString, _ := protojson.Marshal(row)
+			fmt.Printf("%s\n", rowString)
+		}
 		return nil
 	},
 }
 
 func init() {
 	listJobsCmd.Flags().StringVar(&host, "host", host, "grip server url")
 	getCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	viewCmd.Flags().StringVar(&host, "host", host, "grip server url")
 	dropCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	submitCmd.Flags().StringVar(&host, "host", host, "grip server url")
+	resumeCmd.Flags().StringVar(&host, "host", host, "grip server url")
 
 	Cmd.AddCommand(listJobsCmd)
 	Cmd.AddCommand(getCmd)
+	Cmd.AddCommand(viewCmd)
 	Cmd.AddCommand(dropCmd)
+	Cmd.AddCommand(submitCmd)
+	Cmd.AddCommand(resumeCmd)
 }
diff --git a/cmd/query/main.go b/cmd/query/main.go
@@ -2,21 +2,20 @@ package query
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"github.com/bmeg/grip/gripql"
 	gripqljs "github.com/bmeg/grip/gripql/javascript"
 	_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
 	_ "github.com/bmeg/grip/jsengine/otto" // import otto so it registers with the driver map
-	"github.com/bmeg/grip/jsengine/underscore"
+	"github.com/bmeg/grip/log"
 	"github.com/bmeg/grip/util/rpc"
-	"github.com/dop251/goja"
 	"github.com/spf13/cobra"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
 var host = "localhost:8202"
+var verbose bool
 
 // Cmd is the declaration of the command line
 var Cmd = &cobra.Command{
@@ -27,61 +26,48 @@ Example:
     grip query example-graph 'V().hasLabel("Variant").out().limit(5)'`,
 	Args: cobra.ExactArgs(2),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		vm := goja.New()
-
-		us, err := underscore.Asset("underscore.js")
-		if err != nil {
-			return fmt.Errorf("failed to load underscore.js")
-		}
-		if _, err := vm.RunString(string(us)); err != nil {
-			return err
-		}
-
-		gripqlString, err := gripqljs.Asset("gripql.js")
-		if err != nil {
-			return fmt.Errorf("failed to load gripql.js")
-		}
-		if _, err := vm.RunString(string(gripqlString)); err != nil {
-			return err
-		}
-
+		graph := args[0]
 		queryString := args[1]
-		val, err := vm.RunString(queryString)
-		if err != nil {
-			return err
-		}
 
-		queryJSON, err := json.Marshal(val)
-		if err != nil {
-			return err
+		if verbose {
+			c := log.DefaultLoggerConfig()
+			c.Level = "debug"
+			log.ConfigureLogger(c)
 		}
 
-		query := gripql.GraphQuery{}
-		err = protojson.Unmarshal(queryJSON, &query)
+		query, err := gripqljs.ParseQuery(queryString)
 		if err != nil {
+			log.Errorf("Parse error: %s", err)
 			return err
 		}
-		query.Graph = args[0]
-
+		query.Graph = graph
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
+			log.Errorf("Connect error: %s", err)
 			return err
 		}
 
-		res, err := conn.Traversal(context.Background(), &query)
+		log.Debugf("Query: %s\n", query.String())
+		res, err := conn.Traversal(context.Background(), query)
 		if err != nil {
+			log.Errorf("Traversal error: %s", err)
 			return err
 		}
 
+		count := uint64(0)
 		for row := range res {
 			rowString, _ := protojson.Marshal(row)
 			fmt.Printf("%s\n", rowString)
+			count++
 		}
+		log.Debugf("rows returned: %d", count)
 		return nil
 	},
 }
 
 func init() {
 	flags := Cmd.Flags()
 	flags.StringVar(&host, "host", host, "grip server url")
+	flags.BoolVar(&verbose, "verbose", verbose, "Verbose")
+
 }
diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 
 	"github.com/bmeg/grip/config"
 	"github.com/bmeg/grip/log"
@@ -18,8 +19,10 @@ import (
 var conf = &config.Config{}
 var configFile string
 var driver = "badger"
+var verbose bool
 
 var endPoints = map[string]string{}
+var endPointConfig = map[string]string{}
 
 var pluginDir = ""
 
@@ -45,7 +48,14 @@ func Run(conf *config.Config, baseDir string) error {
 	}
 
 	for k, v := range endPoints {
-		if err := srv.AddEndpoint(k, v); err != nil {
+		c := map[string]string{}
+		for ck, cv := range endPointConfig {
+			if strings.HasPrefix(ck, k+":") {
+				nk := ck[len(k)+1:]
+				c[nk] = cv
+			}
+		}
+		if err := srv.AddEndpoint(k, v, c); err != nil {
 			log.Errorf("Error loading pluging %s: %s", k, err)
 		}
 	}
@@ -81,6 +91,8 @@ var Cmd = &cobra.Command{
 				dconf.AddPebbleDefault()
 			} else if driver == "mongo" {
 				dconf.AddMongoDefault()
+			} else if driver == "grids" {
+				dconf.AddGridsDefault()
 			}
 		}
 		if pluginDir != "" {
@@ -100,6 +112,11 @@ var Cmd = &cobra.Command{
 				conf.RPCClient.ServerAddress = conf.Server.RPCAddress()
 			}
 		}
+
+		if verbose {
+			conf.Logger.Level = "debug"
+		}
+
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -117,10 +134,13 @@ func init() {
 	flags.StringVar(&conf.Logger.Formatter, "log-format", conf.Logger.Formatter, "Log format [text, json]")
 	flags.BoolVar(&conf.Server.RequestLogging.Enable, "log-requests", conf.Server.RequestLogging.Enable, "Log all requests")
 
+	flags.BoolVar(&verbose, "verbose", verbose, "Verbose")
+
 	flags.StringVarP(&pluginDir, "plugins", "p", pluginDir, "Directory with GRIPPER plugins")
 	flags.StringVarP(&driver, "driver", "d", driver, "Default Driver")
 
 	flags.StringToStringVarP(&endPoints, "endpoint", "w", endPoints, "web endpoint plugins")
+	flags.StringToStringVarP(&endPointConfig, "endpoint-config", "l", endPointConfig, "web endpoint configuration")
 
 	flags.StringToStringVarP(&conf.Sources, "er", "e", conf.Sources, "GRIPPER source address")
 }