Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixing issues around mongo engine #303

Merged
merged 5 commits into from
Feb 19, 2024
Merged

Fixing issues around mongo engine #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5357f73
Fixing issues around mongo engine
kellrott Feb 17, 2024
69c4867
Fixing additional issues to clear errors in mongo engine
kellrott Feb 18, 2024
942105b
Adding path conversion unit tests
kellrott Feb 18, 2024
0f84a35
Improving command line job submission utilities and fixing mongo core…
kellrott Feb 19, 2024
166f1f7
Fixing compiler options so job system can override data loading optim…
kellrott Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 69 additions & 32 deletions cmd/job/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
package job


import (
"fmt"
"encoding/json"

"github.com/bmeg/grip/gripql"
gripqljs "github.com/bmeg/grip/gripql/javascript"
_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
"github.com/bmeg/grip/util/rpc"
"github.com/spf13/cobra"
_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
"google.golang.org/protobuf/encoding/protojson"
"github.com/dop251/goja"
gripqljs "github.com/bmeg/grip/gripql/javascript"
"github.com/bmeg/grip/jsengine/underscore"
)

var host = "localhost:8202"
Expand All @@ -28,7 +25,7 @@ var listJobsCmd = &cobra.Command{
Long: ``,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
graph := args[0]

conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
Expand All @@ -48,13 +45,13 @@ var listJobsCmd = &cobra.Command{
}

var dropCmd = &cobra.Command{
Use: "drop",
Short: "List graphs",
Use: "drop <job>",
Short: "Drop job",
Long: ``,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
jobID := args[1]
graph := args[0]
jobID := args[1]

conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
Expand All @@ -65,19 +62,19 @@ var dropCmd = &cobra.Command{
if err != nil {
return err
}
fmt.Printf("%s\n", resp)
fmt.Printf("%s\n", resp)
return nil
},
}

var getCmd = &cobra.Command{
Use: "get job",
Use: "get <job>",
Short: "Get job info",
Long: ``,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
jobID := args[1]
graph := args[0]
jobID := args[1]

conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
Expand All @@ -103,70 +100,110 @@ var getCmd = &cobra.Command{
},
}


var submitCmd = &cobra.Command{
Use: "submit <graph> <query>",
Short: "Submit query job",
Long: ``,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
graph := args[0]
queryString := args[1]

vm := goja.New()
us, err := underscore.Asset("underscore.js")
query, err := gripqljs.ParseQuery(queryString)
if err != nil {
return fmt.Errorf("failed to load underscore.js")
}
if _, err := vm.RunString(string(us)); err != nil {
return err
}
gripqlString, err := gripqljs.Asset("gripql.js")
query.Graph = graph

conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
return fmt.Errorf("failed to load gripql.js")
return err
}
if _, err := vm.RunString(string(gripqlString)); err != nil {

res, err := conn.Submit(query)
if err != nil {
return err
}

val, err := vm.RunString(queryString)
fmt.Printf("%s\n", res)
return nil
},
}

var resumeCmd = &cobra.Command{
Use: "resume <graph> <job> <query>",
Short: "Resume query job",
Long: ``,
Args: cobra.ExactArgs(3),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
jobID := args[1]
queryString := args[2]

query, err := gripqljs.ParseQuery(queryString)
if err != nil {
return err
}
query.Graph = graph

queryJSON, err := json.Marshal(val)
conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
return err
}

query := gripql.GraphQuery{}
err = protojson.Unmarshal(queryJSON, &query)
res, err := conn.ResumeJob(graph, jobID, query)
if err != nil {
return err
}
query.Graph = graph

for row := range res {
rowString, _ := protojson.Marshal(row)
fmt.Printf("%s\n", rowString)
}
return nil

},
}

var viewCmd = &cobra.Command{
Use: "view <graph> <job>",
Short: "Resume query job",
Long: ``,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
graph := args[0]
jobID := args[1]

conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
return err
}

res, err := conn.Submit(&query)
res, err := conn.ViewJob(graph, jobID)
if err != nil {
return err
}

fmt.Printf("%s\n", res)
for row := range res {
rowString, _ := protojson.Marshal(row)
fmt.Printf("%s\n", rowString)
}
return nil
},
}

func init() {
listJobsCmd.Flags().StringVar(&host, "host", host, "grip server url")
getCmd.Flags().StringVar(&host, "host", host, "grip server url")
viewCmd.Flags().StringVar(&host, "host", host, "grip server url")
dropCmd.Flags().StringVar(&host, "host", host, "grip server url")
submitCmd.Flags().StringVar(&host, "host", host, "grip server url")
resumeCmd.Flags().StringVar(&host, "host", host, "grip server url")

Cmd.AddCommand(listJobsCmd)
Cmd.AddCommand(getCmd)
Cmd.AddCommand(viewCmd)
Cmd.AddCommand(dropCmd)
Cmd.AddCommand(submitCmd)
Cmd.AddCommand(resumeCmd)
}
39 changes: 4 additions & 35 deletions cmd/query/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
package query

import (
"encoding/json"
"fmt"

"github.com/bmeg/grip/gripql"
gripqljs "github.com/bmeg/grip/gripql/javascript"
_ "github.com/bmeg/grip/jsengine/goja" // import goja so it registers with the driver map
_ "github.com/bmeg/grip/jsengine/otto" // import otto so it registers with the driver map
"github.com/bmeg/grip/jsengine/underscore"
"github.com/bmeg/grip/util/rpc"
"github.com/dop251/goja"
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
)
Expand All @@ -26,48 +23,20 @@ Example:
grip query example-graph 'V().hasLabel("Variant").out().limit(5)'`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
vm := goja.New()

us, err := underscore.Asset("underscore.js")
if err != nil {
return fmt.Errorf("failed to load underscore.js")
}
if _, err := vm.RunString(string(us)); err != nil {
return err
}

gripqlString, err := gripqljs.Asset("gripql.js")
if err != nil {
return fmt.Errorf("failed to load gripql.js")
}
if _, err := vm.RunString(string(gripqlString)); err != nil {
return err
}

graph := args[0]
queryString := args[1]
val, err := vm.RunString(queryString)
if err != nil {
return err
}

queryJSON, err := json.Marshal(val)
query, err := gripqljs.ParseQuery(queryString)
if err != nil {
return err
}

query := gripql.GraphQuery{}
err = protojson.Unmarshal(queryJSON, &query)
if err != nil {
return err
}
query.Graph = args[0]

query.Graph = graph
conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
if err != nil {
return err
}

res, err := conn.Traversal(&query)
res, err := conn.Traversal(query)
if err != nil {
return err
}
Expand Down
6 changes: 3 additions & 3 deletions conformance/tests/ot_aggregations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ def test_field_aggregation(man):
errors = []

# TODO: find way to get gripper driver to drop id field
fields = [ "id", "_gid", "_label", 'orbital_period', 'gravity', 'terrain', 'name','climate', 'system', 'diameter', 'rotation_period', 'url', 'population', 'surface_water']
fields = [ "_id", "id", "_gid", "_label", 'orbital_period', 'gravity', 'terrain', 'name','climate', 'system', 'diameter', 'rotation_period', 'url', 'population', 'surface_water']

G = man.setGraph("swapi")
count = 0
Expand All @@ -201,8 +201,8 @@ def test_field_aggregation(man):
if row["value"] != 3:
errors.append("incorrect count returned: %s" % (row['value']))
count += 1
if count not in [11, 12]: # gripper returns an id field as well, others dont....
errors.append("Incorrect number of results returned")
if count not in [11, 12, 13]: # gripper returns an id field as well, others dont....
errors.append("""V().hasLabel("Planet").aggregate(gripql.field("gid-agg", "$")) : Incorrect number of results returned %d""" % (count))
return errors


Expand Down
6 changes: 3 additions & 3 deletions conformance/tests/ot_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,9 @@ def test_out_edge_out_all(man):
G = man.setGraph("swapi")
for i in G.query().V().as_("a").outE().as_("b").out().as_("c").render(["$a._gid", "$b._from", "$b._to", "$c._gid"]):
if i[0] != i[1]:
errors.append("outE-out _gid/from missmatch %s != %s" % (i[0], i[1]))
errors.append("outE-out _gid/from missmatch '%s' != '%s'" % (i[0], i[1]))
if i[2] != i[3]:
errors.append("outE-out to/_gid missmatch %s != %s" % (i[0], i[1]))
errors.append("outE-out to/_gid missmatch '%s' != '%s'" % (i[2], i[3]))
return errors


Expand Down Expand Up @@ -435,7 +435,7 @@ def test_both_edge(man):

c = G.query().V("Character:1").bothE(["homeworld", "residents"]).count().execute()[0]["count"]
if c != 2:
errors.append("Fail: G.query().V(\"Character:1\").inE([\"homeworld\", \"residents\"]).count() - %s != %s" % (c, 2))
errors.append("Fail: G.query().V(\"Character:1\").bothE([\"homeworld\", \"residents\"]).count() - %s != %s" % (c, 2))

return errors

Expand Down
12 changes: 6 additions & 6 deletions conformance/tests/ot_distinct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,25 @@ def test_distinct(man):
for i in G.query().V().distinct():
count += 1
if count != 39:
errors.append("Distinct %s != %s" % (count, 39))
errors.append("V().distinct() distinct count %s != %s" % (count, 39))

count = 0
for i in G.query().V().distinct("_gid"):
count += 1
if count != 39:
errors.append("Distinct %s != %s" % (count, 39))
errors.append("""V().distinct("_gid") distinct count %s != %s""" % (count, 39))

count = 0
for i in G.query().V().distinct("eye_color"):
count += 1
if count != 8:
errors.append("Distinct %s != %s" % (count, 8))
errors.append("""V().distinct("eye_color") distinct count %s != %s""" % (count, 8))

count = 0
for i in G.query().V().distinct("gender"):
count += 1
if count != 4:
errors.append("Distinct %s != %s" % (count, 4))
errors.append("""V().distinct("gender") distinct count %s != %s""" % (count, 4))

count = 0
for i in G.query().V().distinct("non-existent-field"):
Expand All @@ -37,13 +37,13 @@ def test_distinct(man):
for i in G.query().V().hasLabel("Character").as_("person").out().distinct("$person.name"):
count += 1
if count != 18:
errors.append("Distinct G.query().V().hasLabel(\"Person\").as_(\"person\").out().distinct(\"$person.name\") %s != %s" % (count, 18))
errors.append("Distinct G.query().V().hasLabel(\"Character\").as_(\"person\").out().distinct(\"$person.name\") %s != %s" % (count, 18))

count = 0
for i in G.query().V().hasLabel("Character").as_("person").out().distinct("$person.eye_color"):
count += 1
if count != 8:
errors.append("Distinct G.query().V().hasLabel(\"Person\").as_(\"person\").out().distinct(\"$person.eye_color\") %s != %s" % (count, 8))
errors.append("Distinct G.query().V().hasLabel(\"Character\").as_(\"person\").out().distinct(\"$person.eye_color\") %s != %s" % (count, 8))

return errors

Expand Down
4 changes: 2 additions & 2 deletions conformance/tests/ot_has.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def test_hasId(man):
errors.append("Wrong vertex returned %s" % (i))
if count != 1:
errors.append(
"Fail: G.query().V().hasId(\"01\") %s != %s" %
"Fail: G.query().V().hasId(\"Character:1\") %s != %s" %
(count, 1))

count = 0
Expand Down Expand Up @@ -314,7 +314,7 @@ def test_has_without(man):
errors.append("Wrong vertex returned %s" % (i))
if count != 35:
errors.append(
"Fail: G.query().V().has(gripql.without(\"occupation\", [\"jedi\", \"sith\"])) %s != %s" %
"""Fail: V().has(gripql.without("eye_color", ["brown"])) %s != %s""" %
(count, 35))

count = 0
Expand Down
4 changes: 2 additions & 2 deletions conformance/tests/ot_path_optimize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@ def test_path_1(man):
errors.append("Wrong label found at end of path: %s" % (res["label"]))
count += 1
if count != 1814:
errors.append("out-out-outE Incorrect vertex count returned: %d != %d" % (count, 1814))
errors.append("""V("Film:1").out().out().outE() Incorrect vertex count returned: %d != %d""" % (count, 1814))

count = 0
for res in G.query().V("Film:1").out().out().outE().out():
count += 1
if count != 1814:
errors.append("out-out-outE-out Incorrect vertex count returned: %d != %d" % (count, 1814))
errors.append(""".V("Film:1").out().out().outE().out() Incorrect vertex count returned: %d != %d""" % (count, 1814))

return errors

Expand Down
Loading
Loading