factor out project_id into an optional extra arg

cmd/jsonload/main.go

@@ -1,6 +1,8 @@
 package jsonload
 
 import (
+	"encoding/json"
+
 	"github.com/bmeg/grip/gripql"
 	"github.com/bmeg/grip/log"
 	"github.com/bmeg/grip/util"
@@ -10,20 +12,29 @@ import (
 
 var host = "localhost:8202"
 var NdJsonFile string
-var workerCount = 1
+var workerCount = 5
 var graph string
-var project_id string
+var ExtraArgs string
 var logRate = 10000
 
 var Cmd = &cobra.Command{
-	Use:   "jsonload <NdJsonFile> <graph> <project_id>",
+	Use:   "jsonload <NdJsonFile> <graph> <ExtraArgs>",
 	Short: "Load, Validate NdJson data into grip graph",
 	Long:  ``,
 	Args:  cobra.ExactArgs(3),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		NdJsonFile = args[0]
 		graph = args[1]
-		project_id = args[2]
+		ExtraArgs = args[2]
+
+		var args_map map[string]any
+		if ExtraArgs != "" {
+			err := json.Unmarshal([]byte(ExtraArgs), &args_map)
+			if err != nil {
+				return err
+			}
+		}
+
 		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
 		if err != nil {
 			return err
@@ -55,7 +66,7 @@ var Cmd = &cobra.Command{
 			wait <- false
 		}()
 
-		jsonChan, err := util.StreamRawJsonFromFile(NdJsonFile, workerCount, graph, project_id)
+		jsonChan, err := util.StreamRawJsonFromFile(NdJsonFile, workerCount, graph, args_map)
 		if err != nil {
 			return err
 		}

conformance/tests/ot_bulk_raw.py

@@ -9,7 +9,7 @@ def test_bulk_add_raw(man):
     # Probably don't want to add a 2MB schema file to the repo so get it via requests instead
     res = requests.get("https://raw.githubusercontent.com/bmeg/iceberg/f1724941fe47df24846135fb515d1b89e791cee3/schemas/graph/graph-fhir.json")
     res.raise_for_status()
-    s = G.addJsonSchema(res.json())
+    G.addJsonSchema(res.json())
 
     bulkRaw = G.bulkAddRaw()
     # fhir data from https://github.com/bmeg/iceberg-schema-tools/tree/main/tests/fixtures/simplify-fhir-hypermedia
@@ -23,8 +23,11 @@ def test_bulk_add_raw(man):
         errors.append(f"Wrong number of errors {len(err['errors'])} != 0")
 
     edge = G.getVertex("838e42fb-a65d-4039-9f83-59c37b1ae889")
-    if "gid" not in edge and edge["gid"] != "838e42fb-a65d-4039-9f83-59c37b1ae889":
-        errors.append(f"bulkraw inserted edge with id 838e42fb-a65d-4039-9f83-59c37b1ae889 not found")
+    if "auth_resource_path" not in edge["data"] or edge["data"]["auth_resource_path"] != "test-data":
+        errors.append("ExtraArg auth_resource_path of value test-data not added to vertex")
+
+    if "gid" not in edge or edge["gid"] != "838e42fb-a65d-4039-9f83-59c37b1ae889":
+        errors.append("bulkraw inserted edge with id 838e42fb-a65d-4039-9f83-59c37b1ae889 not found")
 
     labels = G.listLabels()
     if not all(item in labels['vertexLabels'] for item in ['Condition', 'Patient']) or not all (item in labels['edgeLabels'] for item in ['condition', 'subject_Patient']):

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad
 	github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92
-	github.com/bmeg/jsonschemagraph v0.0.3-0.20241113190142-5e57a1561020
+	github.com/bmeg/jsonschemagraph v0.0.3-0.20241130225037-5545ec0ffd4b
 	github.com/boltdb/bolt v1.3.1
 	github.com/casbin/casbin/v2 v2.97.0
 	github.com/cockroachdb/pebble v1.1.1
@@ -58,6 +58,7 @@ require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/alevinval/sse v1.0.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc // indirect
 	github.com/casbin/govaluate v1.2.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect

go.sum

@@ -29,12 +29,18 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc h1:/0v/ZcXYjGs44InmjECrls31onIbVKVu1Q/E2cmnCEU=
+github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc/go.mod h1:hoSeuZtqe58ANXHuWpeODx4bDHGV36QXlCs1yrUvK6M=
 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8=
 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4=
 github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU=
 github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92/go.mod h1:6v27bSBKXyIDFqlKQbUSnHlekE1y6bDkgWCuVEaDPng=
 github.com/bmeg/jsonschemagraph v0.0.3-0.20241113190142-5e57a1561020 h1:7/dWlBDJdKYhtF31LO8zXcw/IcYsmp/MWpydk6PzuNA=
 github.com/bmeg/jsonschemagraph v0.0.3-0.20241113190142-5e57a1561020/go.mod h1:k04v50661tQFKgYl6drQxWGf9q0dBmKhd6lwIk1rcCw=
+github.com/bmeg/jsonschemagraph v0.0.3-0.20241130214835-4863185c5c72 h1:p8LS6V7vX/Hq+cJl8WvEQZjN2X+xjIXQTuQkstgfG6I=
+github.com/bmeg/jsonschemagraph v0.0.3-0.20241130214835-4863185c5c72/go.mod h1:k04v50661tQFKgYl6drQxWGf9q0dBmKhd6lwIk1rcCw=
+github.com/bmeg/jsonschemagraph v0.0.3-0.20241130225037-5545ec0ffd4b h1:k9j0qId2gRBwYhSdvqlBfQQDNIU4nohG4J9lu9crWE0=
+github.com/bmeg/jsonschemagraph v0.0.3-0.20241130225037-5545ec0ffd4b/go.mod h1:k04v50661tQFKgYl6drQxWGf9q0dBmKhd6lwIk1rcCw=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=