From ac185a097d7442012a72f2a8f1054fc4630b2b70 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Thu, 21 Mar 2024 14:11:59 -0700 Subject: [PATCH] Fixing go mod links --- go.mod | 41 +- go.sum | 93 ++-- grids/old/compiler.go | 114 ----- grids/old/config.go | 5 - grids/old/graph.go | 813 -------------------------------- grids/old/graph_lite.go | 1 - grids/old/graphdb.go | 71 --- grids/old/index.go | 114 ----- grids/old/keymap.go | 284 ----------- grids/old/keys.go | 166 ------- grids/old/new.go | 105 ----- grids/old/raw_graph.go | 262 ---------- grids/old/raw_processors.go | 301 ------------ grids/old/schema.go | 91 ---- grids/old/statement_compiler.go | 250 ---------- grids/old/traveler.go | 269 ----------- server/server.go | 2 +- 17 files changed, 67 insertions(+), 2915 deletions(-) delete mode 100644 grids/old/compiler.go delete mode 100644 grids/old/config.go delete mode 100644 grids/old/graph.go delete mode 100644 grids/old/graph_lite.go delete mode 100644 grids/old/graphdb.go delete mode 100644 grids/old/index.go delete mode 100644 grids/old/keymap.go delete mode 100644 grids/old/keys.go delete mode 100644 grids/old/new.go delete mode 100644 grids/old/raw_graph.go delete mode 100644 grids/old/raw_processors.go delete mode 100644 grids/old/schema.go delete mode 100644 grids/old/statement_compiler.go delete mode 100644 grids/old/traveler.go diff --git a/go.mod b/go.mod index f360852f..2c48b4c3 100644 --- a/go.mod +++ b/go.mod @@ -44,19 +44,19 @@ require ( github.com/stretchr/testify v1.8.2 github.com/syndtr/goleveldb v1.0.0 go.mongodb.org/mongo-driver v1.12.0 - golang.org/x/crypto v0.6.0 - golang.org/x/net v0.7.0 - golang.org/x/sync v0.1.0 - google.golang.org/api v0.110.0 - google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743 + golang.org/x/crypto v0.18.0 + golang.org/x/net v0.20.0 + golang.org/x/sync v0.6.0 + google.golang.org/api v0.149.0 + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 + google.golang.org/grpc v1.62.1 + google.golang.org/protobuf v1.33.0 gopkg.in/olivere/elastic.v5 v5.0.80 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect @@ -80,13 +80,14 @@ require ( github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-hclog v0.14.1 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect @@ -108,7 +109,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/oklog/run v1.0.0 // indirect github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -127,14 +127,15 @@ require ( github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect - golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.8.2 // indirect - google.golang.org/appengine v1.6.7 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a415ba15..b382fe13 100644 --- a/go.sum +++ b/go.sum @@ -13,21 +13,19 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -150,13 +148,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg= -github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dop251/goja v0.0.0-20190429205339-8d6ee3d16611 h1:1MBTvhEvrqHgSDeoxWrBgDt+ndbmURmCgFZL7f6OXk4= -github.com/dop251/goja v0.0.0-20190429205339-8d6ee3d16611/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 h1:O7I1iuzEA7SG+dK8ocOBSlYAA9jBUmCYl/Qa7ey7JAM= github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= @@ -209,8 +203,6 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= -github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -254,12 +246,13 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -283,8 +276,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -303,7 +297,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -318,15 +312,17 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -447,7 +443,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -534,8 +529,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nsf/termbox-go v0.0.0-20160718140619-0723e7c3d0a3/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= @@ -732,8 +725,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -814,16 +807,16 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -836,8 +829,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -899,12 +892,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -913,8 +906,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1004,16 +998,16 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= -google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1045,8 +1039,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos= -google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1062,8 +1060,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1077,14 +1075,13 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743 h1:yqElulDvOF26oZ2O+2/aoX7mQ8DY/6+p39neytrycd8= -google.golang.org/protobuf v1.28.2-0.20230222093303-bc1253ad3743/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/grids/old/compiler.go b/grids/old/compiler.go deleted file mode 100644 index b6ad1f72..00000000 --- a/grids/old/compiler.go +++ /dev/null @@ -1,114 +0,0 @@ -package grids - -import ( - "fmt" - - "github.com/bmeg/grip/engine/core" - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/util/protoutil" -) - -// Compiler gets a compiler that will use the graph the execute the compiled query -func (ggraph *Graph) Compiler() gdbi.Compiler { - return NewCompiler(ggraph) -} - -type Compiler struct { - graph *Graph -} - -func SelectPath(stmts []*gripql.GraphStatement, path []int) []*gripql.GraphStatement { - out := []*gripql.GraphStatement{} - for _, p := range path { - out = append(out, stmts[p]) - } - return out -} - -func NewCompiler(ggraph *Graph) gdbi.Compiler { - return core.NewCompiler(ggraph, core.IndexStartOptimize) - //return Compiler{graph: ggraph} -} - -func (comp Compiler) Compile(stmts []*gripql.GraphStatement, opts *gdbi.CompileOptions) (gdbi.Pipeline, error) { - fmt.Printf("Doing compile\n") - if len(stmts) == 0 { - return &core.DefaultPipeline{}, nil - } - - if err := core.Validate(stmts, opts); err != nil { - fmt.Printf("Failing validation\n") - return &core.DefaultPipeline{}, fmt.Errorf("invalid statments: %s", err) - } - - stmts = core.IndexStartOptimize(stmts) - - ps := gdbi.NewPipelineState(stmts) - if opts != nil { - ps.LastType = opts.PipelineExtension - ps.MarkTypes = opts.ExtensionMarkTypes - } - fmt.Printf("GRIDS compile: %#v %#v\n", *ps, opts) - - procs := make([]gdbi.Processor, 0, len(stmts)) - - optimizeOn := false - - fmt.Printf("Starting Grids Compiler: %s\n", stmts) - for i := 0; i < len(stmts); i++ { - gs := stmts[i] - ps.SetCurStatment(i) - if p, err := GetRawProcessor(comp.graph, ps, gs); err == nil && optimizeOn { - procs = append(procs, p) - } else { - p, err := gdbi.StatementProcessor(gs, comp.graph, ps) - if err != nil { - fmt.Printf("Error %s at %d %#v", err, i, gs) - return &core.DefaultPipeline{}, err - } - procs = append(procs, p) - } - } - fmt.Printf("GRIDS: Pipeline: %#v\n", procs) - return core.NewPipeline(comp.graph, procs, ps), nil -} - -func GetRawProcessor(db *Graph, ps gdbi.PipelineState, stmt *gripql.GraphStatement) (gdbi.Processor, error) { - switch stmt := stmt.GetStatement().(type) { - case *gripql.GraphStatement_V: - ids := protoutil.AsStringList(stmt.V) - ps.SetLastType(gdbi.VertexData) - return &PathVProc{db: db, ids: ids}, nil - case *gripql.GraphStatement_In: - if ps.GetLastType() == gdbi.VertexData { - labels := protoutil.AsStringList(stmt.In) - ps.SetLastType(gdbi.VertexData) - return &PathInProc{db: db, labels: labels}, nil - } else if ps.GetLastType() == gdbi.EdgeData { - ps.SetLastType(gdbi.VertexData) - return &PathInEdgeAdjProc{db: db}, nil - } - case *gripql.GraphStatement_Out: - if ps.GetLastType() == gdbi.VertexData { - labels := protoutil.AsStringList(stmt.Out) - ps.SetLastType(gdbi.VertexData) - return &PathOutProc{db: db, labels: labels}, nil - } else if ps.GetLastType() == gdbi.EdgeData { - ps.SetLastType(gdbi.VertexData) - return &PathOutEdgeAdjProc{db: db}, nil - } - case *gripql.GraphStatement_InE: - labels := protoutil.AsStringList(stmt.InE) - ps.SetLastType(gdbi.EdgeData) - return &PathInEProc{db: db, labels: labels}, nil - case *gripql.GraphStatement_OutE: - labels := protoutil.AsStringList(stmt.OutE) - ps.SetLastType(gdbi.EdgeData) - return &PathOutEProc{db: db, labels: labels}, nil - case *gripql.GraphStatement_HasLabel: - labels := protoutil.AsStringList(stmt.HasLabel) - return &PathLabelProc{db: db, labels: labels}, nil - } - return nil, fmt.Errorf("unknown command: %T", stmt) -} diff --git a/grids/old/config.go b/grids/old/config.go deleted file mode 100644 index 8cddba33..00000000 --- a/grids/old/config.go +++ /dev/null @@ -1,5 +0,0 @@ -package grids - -type Config struct { - Path string `json:"path"` -} diff --git a/grids/old/graph.go b/grids/old/graph.go deleted file mode 100644 index 94095692..00000000 --- a/grids/old/graph.go +++ /dev/null @@ -1,813 +0,0 @@ -package grids - -import ( - "bytes" - "context" - "fmt" - "sync" - - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/kvi" - "github.com/bmeg/grip/kvindex" - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/util/protoutil" - "github.com/bmeg/grip/util/setcmp" - multierror "github.com/hashicorp/go-multierror" -) - -// GetTimestamp returns the update timestamp -func (ggraph *Graph) GetTimestamp() string { - //return ggraph.kdb.ts.Get(ggraph.graphID) - return "" //FIXME -} - -type kvAddData struct { - key []byte - value []byte - vertex *gripql.Vertex - doc map[string]interface{} -} - -func insertVertex(tx kvi.KVBulkWrite, keyMap *KeyMap, vertex *gdbi.Vertex) error { - if vertex.ID == "" { - return fmt.Errorf("Inserting null key vertex") - } - vertexKey, _ := keyMap.GetsertVertexKey(vertex.ID, vertex.Label) - key := VertexKey(vertexKey) - if vertex.Data == nil { - vertex.Data = map[string]interface{}{} - } - value, err := protoutil.StructMarshal(vertex.Data) - if err != nil { - return err - } - if err := tx.Set(key, value); err != nil { - return fmt.Errorf("AddVertex Error %s", err) - } - return nil -} - -func indexVertex(tx kvi.KVBulkWrite, idx *kvindex.KVIndex, vertex *gdbi.Vertex) error { - doc := vertexIdxStruct(vertex) - if err := idx.AddDocTx(tx, vertex.ID, doc); err != nil { - return fmt.Errorf("AddVertex Error %s", err) - } - return nil -} - -func insertEdge(tx kvi.KVBulkWrite, keyMap *KeyMap, edge *gdbi.Edge) error { - var err error - var data []byte - - if edge.ID == "" { - return fmt.Errorf("inserting null key edge") - } - - eid, lid := keyMap.GetsertEdgeKey(edge.ID, edge.Label) - src, ok := keyMap.GetVertexKey(edge.From) - if !ok { - return fmt.Errorf("vertex %s not found", edge.From) - } - dst, ok := keyMap.GetVertexKey(edge.To) - if !ok { - return fmt.Errorf("vertex %s not found", edge.To) - } - - ekey := EdgeKey(eid, src, dst, lid) - skey := SrcEdgeKey(eid, src, dst, lid) - dkey := DstEdgeKey(eid, src, dst, lid) - - data, err = protoutil.StructMarshal(edge.Data) - if err != nil { - return err - } - - err = tx.Set(ekey, data) - if err != nil { - return err - } - err = tx.Set(skey, []byte{}) - if err != nil { - return err - } - err = tx.Set(dkey, []byte{}) - if err != nil { - return err - } - return nil -} - -func indexEdge(tx kvi.KVBulkWrite, idx *kvindex.KVIndex, edge *gdbi.Edge) error { - err := idx.AddDocTx(tx, edge.ID, edgeIdxStruct(edge)) - return err -} - -// AddVertex adds an edge to the graph, if it already exists -// in the graph, it is replaced -func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { - err := ggraph.graphkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - var bulkErr *multierror.Error - for _, vert := range vertices { - if err := insertVertex(tx, ggraph.keyMap, vert); err != nil { - bulkErr = multierror.Append(bulkErr, err) - log.Errorf("AddVertex Error %s", err) - } - } - ggraph.ts.Touch(ggraph.graphID) - return bulkErr.ErrorOrNil() - }) - err = ggraph.indexkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - var bulkErr *multierror.Error - for _, vert := range vertices { - if err := indexVertex(tx, ggraph.idx, vert); err != nil { - bulkErr = multierror.Append(bulkErr, err) - log.Errorf("IndexVertex Error %s", err) - } - } - ggraph.ts.Touch(ggraph.graphID) - return bulkErr.ErrorOrNil() - }) - return err -} - -// AddEdge adds an edge to the graph, if the id is not "" and in already exists -// in the graph, it is replaced -func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { - err := ggraph.graphkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - for _, edge := range edges { - err := insertEdge(tx, ggraph.keyMap, edge) - if err != nil { - return err - } - } - ggraph.ts.Touch(ggraph.graphID) - return nil - }) - if err != nil { - return err - } - err = ggraph.indexkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - var bulkErr *multierror.Error - for _, edge := range edges { - if err := indexEdge(tx, ggraph.idx, edge); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - ggraph.ts.Touch(ggraph.graphID) - return bulkErr.ErrorOrNil() - }) - return err - -} - -func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { - var anyErr error - insertStream := make(chan *gdbi.GraphElement, 100) - indexStream := make(chan *gdbi.GraphElement, 100) - s := &sync.WaitGroup{} - s.Add(2) - go func() { - ggraph.graphkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - for elem := range insertStream { - if elem.Vertex != nil { - if err := insertVertex(tx, ggraph.keyMap, elem.Vertex); err != nil { - anyErr = err - } - } - if elem.Edge != nil { - if err := insertEdge(tx, ggraph.keyMap, elem.Edge); err != nil { - anyErr = err - } - } - } - s.Done() - return anyErr - }) - }() - - go func() { - ggraph.indexkv.BulkWrite(func(tx kvi.KVBulkWrite) error { - for elem := range indexStream { - if elem.Vertex != nil { - if err := indexVertex(tx, ggraph.idx, elem.Vertex); err != nil { - anyErr = err - } - } - if elem.Edge != nil { - if err := indexEdge(tx, ggraph.idx, elem.Edge); err != nil { - anyErr = err - } - } - } - s.Done() - return anyErr - }) - }() - - for i := range stream { - insertStream <- i - indexStream <- i - } - close(insertStream) - close(indexStream) - s.Wait() - return anyErr -} - -// DelEdge deletes edge with id `key` -func (ggraph *Graph) DelEdge(eid string) error { - edgeKey, ok := ggraph.keyMap.GetEdgeKey(eid) - if !ok { - return fmt.Errorf("edge not found") - } - ekeyPrefix := EdgeKeyPrefix(edgeKey) - var ekey []byte - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - ekey = it.Key() - } - return nil - }) - - if ekey == nil { - return fmt.Errorf("edge not found") - } - - _, sid, did, _ := EdgeKeyParse(ekey) - - skey := SrcEdgeKeyPrefix(edgeKey, sid, did) - dkey := DstEdgeKeyPrefix(edgeKey, sid, did) - - var bulkErr *multierror.Error - if err := ggraph.graphkv.Delete(ekey); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - if err := ggraph.graphkv.Delete(skey); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - if err := ggraph.graphkv.Delete(dkey); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - if err := ggraph.keyMap.DelEdgeKey(eid); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - ggraph.ts.Touch(ggraph.graphID) - return bulkErr.ErrorOrNil() -} - -// DelVertex deletes vertex with id `key` -func (ggraph *Graph) DelVertex(id string) error { - vertexKey, ok := ggraph.keyMap.GetVertexKey(id) - if !ok { - return fmt.Errorf("vertex %s not found", id) - } - vid := VertexKey(vertexKey) - skeyPrefix := SrcEdgePrefix(vertexKey) - dkeyPrefix := DstEdgePrefix(vertexKey) - - delKeys := make([][]byte, 0, 1000) - - var bulkErr *multierror.Error - - err := ggraph.graphkv.View(func(it kvi.KVIterator) error { - var bulkErr *multierror.Error - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - skey := it.Key() - // get edge ID from key - eid, sid, did, label := SrcEdgeKeyParse(skey) - ekey := EdgeKey(eid, sid, did, label) - dkey := DstEdgeKey(eid, sid, did, label) - delKeys = append(delKeys, ekey, skey, dkey) - - edgeID, ok := ggraph.keyMap.GetEdgeID(eid) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - } - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - dkey := it.Key() - // get edge ID from key - eid, sid, did, label := SrcEdgeKeyParse(dkey) - ekey := EdgeKey(eid, sid, did, label) - skey := SrcEdgeKey(eid, sid, did, label) - delKeys = append(delKeys, ekey, skey, dkey) - - edgeID, ok := ggraph.keyMap.GetEdgeID(eid) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - } - return bulkErr.ErrorOrNil() - }) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - - if err := ggraph.keyMap.DelVertexKey(id); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - - err = ggraph.graphkv.Update(func(tx kvi.KVTransaction) error { - if err := tx.Delete(vid); err != nil { - return err - } - for _, k := range delKeys { - if err := tx.Delete(k); err != nil { - return err - } - } - ggraph.ts.Touch(ggraph.graphID) - return nil - }) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - return bulkErr.ErrorOrNil() -} - -// GetEdgeList produces a channel of all edges in the graph -func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - ePrefix := EdgeListPrefix() - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - keyValue := it.Key() - ekey, skey, dkey, label := EdgeKeyParse(keyValue) - labelID, _ := ggraph.keyMap.GetLabelID(label) - sid, _ := ggraph.keyMap.GetVertexID(skey) - did, _ := ggraph.keyMap.GetVertexID(dkey) - eid, _ := ggraph.keyMap.GetEdgeID(ekey) - e := &gdbi.Edge{ID: eid, Label: labelID, From: sid, To: did} - if loadProp { - var err error - edgeData, _ := it.Value() - e.Data, err = protoutil.StructUnMarshal(edgeData) - e.Loaded = true - if err != nil { - log.Errorf("GetEdgeList: unmarshal error: %v", err) - continue - } - } else { - e.Data = map[string]interface{}{} - } - o <- e - } - return nil - }) - }() - return o -} - -// GetVertex loads a vertex given an id. It returns a nil if not found -func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { - key, ok := ggraph.keyMap.GetVertexKey(id) - if !ok { - return nil - } - vkey := VertexKey(key) - - var v *gdbi.Vertex - err := ggraph.graphkv.View(func(it kvi.KVIterator) error { - lKey := ggraph.keyMap.GetVertexLabel(key) - lID, _ := ggraph.keyMap.GetLabelID(lKey) - v = &gdbi.Vertex{ - ID: id, - Label: lID, - } - if loadProp { - dataValue, err := it.Get(vkey) - v.Data, err = protoutil.StructUnMarshal(dataValue) - v.Loaded = true - if err != nil { - return fmt.Errorf("unmarshal error: %v", err) - } - } else { - v.Data = map[string]interface{}{} - } - return nil - }) - if err != nil { - return nil - } - return v -} - -type elementData struct { - key uint64 - req gdbi.ElementLookup - data []byte -} - -// GetVertexChannel is passed a channel of vertex ids and it produces a channel -// of vertices -func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { - data := make(chan elementData, 100) - go func() { - defer close(data) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for id := range ids { - if id.IsSignal() { - data <- elementData{req: id} - } else { - key, _ := ggraph.keyMap.GetVertexKey(id.ID) - ed := elementData{key: key, req: id} - if load { - vkey := VertexKey(key) - dataValue, err := it.Get(vkey) - if err == nil { - ed.data = dataValue - } - } - data <- ed - } - } - return nil - }) - }() - - out := make(chan gdbi.ElementLookup, 100) - go func() { - defer close(out) - for d := range data { - if d.req.IsSignal() { - out <- d.req - } else { - lKey := ggraph.keyMap.GetVertexLabel(d.key) - lID, _ := ggraph.keyMap.GetLabelID(lKey) - v := gdbi.Vertex{ID: d.req.ID, Label: lID} - if load { - var err error - v.Data, err = protoutil.StructUnMarshal(d.data) - if err != nil { - log.Errorf("GetVertexChannel: unmarshal error: %v", err) - continue - } - v.Loaded = true - } else { - v.Data = map[string]interface{}{} - } - d.req.Vertex = &v - out <- d.req - } - } - }() - - return out -} - -// GetOutChannel process requests of vertex ids and find the connected vertices on outgoing edges -func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - vertexChan := make(chan elementData, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(vertexChan) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - vertexChan <- elementData{req: req} - } else { - key, ok := ggraph.keyMap.GetVertexKey(req.ID) - if ok { - skeyPrefix := SrcEdgePrefix(key) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - _, _, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - vkey := VertexKey(dst) - vertexChan <- elementData{ - data: vkey, - req: req, - } - } - } - } - } - } - return nil - }) - }() - - o := make(chan gdbi.ElementLookup, 100) - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range vertexChan { - if req.req.IsSignal() { - o <- req.req - } else { - vkey := VertexKeyParse(req.data) - gid, _ := ggraph.keyMap.GetVertexID(vkey) - lkey := ggraph.keyMap.GetVertexLabel(vkey) - lid, _ := ggraph.keyMap.GetLabelID(lkey) - v := &gdbi.Vertex{ID: gid, Label: lid} - if load { - dataValue, err := it.Get(req.data) - if err == nil { - v.Data, err = protoutil.StructUnMarshal(dataValue) - if err != nil { - log.Errorf("GetOutChannel: unmarshal error: %v", err) - continue - } - v.Loaded = true - } - } else { - v.Data = map[string]interface{}{} - } - req.req.Vertex = v - o <- req.req - } - } - return nil - }) - }() - return o -} - -// GetInChannel process requests of vertex ids and find the connected vertices on incoming edges -func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID) - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - _, src, _, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - vkey := VertexKey(src) - srcID, _ := ggraph.keyMap.GetVertexID(src) - lKey := ggraph.keyMap.GetVertexLabel(src) - lID, _ := ggraph.keyMap.GetLabelID(lKey) - v := &gdbi.Vertex{ID: srcID, Label: lID} - if load { - dataValue, err := it.Get(vkey) - if err == nil { - v.Data, err = protoutil.StructUnMarshal(dataValue) - if err != nil { - log.Errorf("GetInChannel: unmarshal error: %v", err) - continue - } - v.Loaded = true - } - } else { - v.Data = map[string]interface{}{} - } - req.Vertex = v - o <- req - } - } - } - } - } - return nil - }) - }() - return o -} - -// GetOutEdgeChannel process requests of vertex ids and find the connected outgoing edges -func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID) - if ok { - skeyPrefix := SrcEdgePrefix(vkey) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid) - e.From, _ = ggraph.keyMap.GetVertexID(src) - e.To, _ = ggraph.keyMap.GetVertexID(dst) - e.Label, _ = ggraph.keyMap.GetLabelID(label) - if load { - ekey := EdgeKey(eid, src, dst, label) - dataValue, err := it.Get(ekey) - if err == nil { - e.Data, err = protoutil.StructUnMarshal(dataValue) - if err != nil { - log.Errorf("GetOutEdgeChannel: unmarshal error: %v", err) - continue - } - e.Loaded = true - } - } else { - e.Data = map[string]interface{}{} - } - req.Edge = &e - o <- req - } - } - } - } - } - return nil - }) - - }() - return o -} - -// GetInEdgeChannel process requests of vertex ids and find the connected incoming edges -func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID) - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid) - e.From, _ = ggraph.keyMap.GetVertexID(src) - e.To, _ = ggraph.keyMap.GetVertexID(dst) - e.Label, _ = ggraph.keyMap.GetLabelID(label) - if load { - ekey := EdgeKey(eid, src, dst, label) - dataValue, err := it.Get(ekey) - if err == nil { - e.Data, err = protoutil.StructUnMarshal(dataValue) - if err != nil { - log.Errorf("GetInEdgeChannel: unmarshal error: %v", err) - continue - } - e.Loaded = true - } - } else { - e.Data = map[string]interface{}{} - } - req.Edge = &e - o <- req - } - } - } - } - } - return nil - }) - - }() - return o -} - -// GetEdge loads an edge given an id. It returns nil if not found -func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { - ekey, ok := ggraph.keyMap.GetEdgeKey(id) - if !ok { - return nil - } - ekeyPrefix := EdgeKeyPrefix(ekey) - - var e *gdbi.Edge - err := ggraph.graphkv.View(func(it kvi.KVIterator) error { - for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - eid, src, dst, labelKey := EdgeKeyParse(it.Key()) - gid, _ := ggraph.keyMap.GetEdgeID(eid) - from, _ := ggraph.keyMap.GetVertexID(src) - to, _ := ggraph.keyMap.GetVertexID(dst) - label, _ := ggraph.keyMap.GetLabelID(labelKey) - e = &gdbi.Edge{ - ID: gid, - From: from, - To: to, - Label: label, - } - if loadProp { - var err error - d, _ := it.Value() - e.Data, err = protoutil.StructUnMarshal(d) - if err != nil { - return fmt.Errorf("unmarshal error: %v", err) - } - e.Loaded = true - } else { - e.Data = map[string]interface{}{} - } - } - return nil - }) - if err != nil { - return nil - } - return e -} - -// GetVertexList produces a channel of all edges in the graph -func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *gdbi.Vertex { - o := make(chan *gdbi.Vertex, 100) - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - vPrefix := VertexListPrefix() - - for it.Seek(vPrefix); it.Valid() && bytes.HasPrefix(it.Key(), vPrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - v := &gdbi.Vertex{} - keyValue := it.Key() - vKey := VertexKeyParse(keyValue) - lKey := ggraph.keyMap.GetVertexLabel(vKey) - v.ID, _ = ggraph.keyMap.GetVertexID(vKey) - v.Label, _ = ggraph.keyMap.GetLabelID(lKey) - if loadProp { - var err error - dataValue, _ := it.Value() - v.Data, err = protoutil.StructUnMarshal(dataValue) - if err != nil { - log.Errorf("GetVertexList: unmarshal error: %v", err) - continue - } - v.Loaded = true - } else { - v.Data = map[string]interface{}{} - } - o <- v - } - return nil - }) - }() - return o -} - -// ListVertexLabels returns a list of vertex types in the graph -func (ggraph *Graph) ListVertexLabels() ([]string, error) { - labelField := fmt.Sprintf("%s.v.label", ggraph.graphID) - labels := []string{} - for i := range ggraph.idx.FieldTerms(labelField) { - labels = append(labels, i.(string)) - } - return labels, nil -} - -// ListEdgeLabels returns a list of edge types in the graph -func (ggraph *Graph) ListEdgeLabels() ([]string, error) { - labelField := fmt.Sprintf("%s.e.label", ggraph.graphID) - labels := []string{} - for i := range ggraph.idx.FieldTerms(labelField) { - labels = append(labels, i.(string)) - } - return labels, nil -} diff --git a/grids/old/graph_lite.go b/grids/old/graph_lite.go deleted file mode 100644 index 6f7f7b1f..00000000 --- a/grids/old/graph_lite.go +++ /dev/null @@ -1 +0,0 @@ -package grids diff --git a/grids/old/graphdb.go b/grids/old/graphdb.go deleted file mode 100644 index c2e9b333..00000000 --- a/grids/old/graphdb.go +++ /dev/null @@ -1,71 +0,0 @@ -package grids - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/log" -) - -// GridsGDB implements the GripInterface using a generic key/value storage driver -type GDB struct { - basePath string - drivers map[string]*Graph -} - -// NewKVGraphDB intitalize a new grids graph driver -func NewGraphDB(baseDir string) (gdbi.GraphDB, error) { - log.Warning("GRIP driver is development. Do not use") - _, err := os.Stat(baseDir) - if os.IsNotExist(err) { - os.Mkdir(baseDir, 0700) - } - return &GDB{basePath: baseDir, drivers: map[string]*Graph{}}, nil -} - -// Graph obtains the gdbi.DBI for a particular graph -func (kgraph *GDB) Graph(graph string) (gdbi.GraphInterface, error) { - err := gripql.ValidateGraphName(graph) - if err != nil { - return nil, err - } - if g, ok := kgraph.drivers[graph]; ok { - return g, nil - } - dbPath := filepath.Join(kgraph.basePath, graph) - if _, err := os.Stat(dbPath); err == nil { - g, err := newGraph(kgraph.basePath, graph) - if err != nil { - return nil, err - } - kgraph.drivers[graph] = g - return g, nil - } - return nil, fmt.Errorf("graph '%s' was not found", graph) -} - -// ListGraphs lists the graphs managed by this driver -func (gdb *GDB) ListGraphs() []string { - out := []string{} - for k := range gdb.drivers { - out = append(out, k) - } - if ds, err := filepath.Glob(filepath.Join(gdb.basePath, "*")); err == nil { - for _, d := range ds { - b := filepath.Base(d) - out = append(out, b) - } - } - return out -} - -// Close the graphs -func (kgraph *GDB) Close() error { - for _, g := range kgraph.drivers { - g.Close() - } - return nil -} diff --git a/grids/old/index.go b/grids/old/index.go deleted file mode 100644 index f802e6a7..00000000 --- a/grids/old/index.go +++ /dev/null @@ -1,114 +0,0 @@ -package grids - -import ( - "context" - "fmt" - "strings" - - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/travelerpath" -) - -func (kgraph *Graph) setupGraphIndex(graph string) error { - err := kgraph.idx.AddField(fmt.Sprintf("%s.v.label", graph)) - if err != nil { - return fmt.Errorf("failed to setup index on vertex label") - } - err = kgraph.idx.AddField(fmt.Sprintf("%s.e.label", graph)) - if err != nil { - return fmt.Errorf("failed to setup index on edge label") - } - return nil -} - -func (kgraph *Graph) deleteGraphIndex(graph string) error { - var anyError error - fields := kgraph.idx.ListFields() - for _, f := range fields { - t := strings.Split(f, ".") - if t[0] == graph { - if err := kgraph.idx.RemoveField(f); err != nil { - anyError = err - } - } - } - return anyError -} - -func normalizePath(path string) string { - path = travelerpath.GetJSONPath(path) - path = strings.TrimPrefix(path, "$.") - path = strings.TrimPrefix(path, "data.") - return path -} - -func vertexIdxStruct(v *gdbi.Vertex) map[string]interface{} { - k := map[string]interface{}{ - "v": map[string]interface{}{ - "label": v.Label, - v.Label: v.Data, - }, - } - return k -} - -func edgeIdxStruct(e *gdbi.Edge) map[string]interface{} { - k := map[string]interface{}{ - "e": map[string]interface{}{ - "label": e.Label, - e.Label: e.Data, - }, - } - return k -} - -// AddVertexIndex add index to vertices -func (ggraph *Graph) AddVertexIndex(label string, field string) error { - log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") - field = normalizePath(field) - //TODO kick off background process to reindex existing data - return ggraph.idx.AddField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) -} - -// DeleteVertexIndex delete index from vertices -func (ggraph *Graph) DeleteVertexIndex(label string, field string) error { - log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - field = normalizePath(field) - return ggraph.idx.RemoveField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) -} - -// GetVertexIndexList lists out all the vertex indices for a graph -func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { - log.Debug("Running GetVertexIndexList") - out := make(chan *gripql.IndexID) - go func() { - defer close(out) - fields := ggraph.idx.ListFields() - for _, f := range fields { - t := strings.Split(f, ".") - if len(t) > 3 { - out <- &gripql.IndexID{Graph: ggraph.graphID, Label: t[2], Field: t[3]} - } - } - }() - return out -} - -// VertexLabelScan produces a channel of all vertex ids in a graph -// that match a given label -func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan string { - log.WithFields(log.Fields{"label": label}).Debug("Running VertexLabelScan") - //TODO: Make this work better - out := make(chan string, 100) - go func() { - defer close(out) - //log.Printf("Searching %s %s", fmt.Sprintf("%s.label", ggraph.graph), label) - for i := range ggraph.idx.GetTermMatch(ctx, fmt.Sprintf("%s.v.label", ggraph.graphID), label, 0) { - //log.Printf("Found: %s", i) - out <- i - } - }() - return out -} diff --git a/grids/old/keymap.go b/grids/old/keymap.go deleted file mode 100644 index 70eaa535..00000000 --- a/grids/old/keymap.go +++ /dev/null @@ -1,284 +0,0 @@ -package grids - -import ( - "bytes" - "encoding/binary" - "fmt" - "sync" - - "github.com/akrylysov/pogreb" - "github.com/bmeg/grip/log" -) - -type KeyMap struct { - db *pogreb.DB - - vIncCur uint64 - eIncCur uint64 - lIncCur uint64 - - vIncMut sync.Mutex - eIncMut sync.Mutex - lIncMut sync.Mutex -} - -var incMod uint64 = 1000 - -var vIDPrefix = []byte{'v'} -var eIDPrefix = []byte{'e'} -var lIDPrefix = []byte{'l'} - -var vKeyPrefix byte = 'V' -var eKeyPrefix byte = 'E' -var lKeyPrefix byte = 'L' - -var vLabelPrefix byte = 'x' -var eLabelPrefix byte = 'y' - -var vInc = []byte{'i', 'v'} -var eInc = []byte{'i', 'e'} -var lInc = []byte{'i', 'l'} - -func NewKeyMap(kv *pogreb.DB) *KeyMap { - return &KeyMap{db: kv} -} - -func (km *KeyMap) Close() { - km.db.Close() -} - -//GetsertVertexKey : Get or Insert Vertex Key -func (km *KeyMap) GetsertVertexKey(id, label string) (uint64, uint64) { - o, ok := getIDKey(vIDPrefix, id, km.db) - if !ok { - km.vIncMut.Lock() - var err error - o, err = dbInc(&km.vIncCur, vInc, km.db) - if err != nil { - log.Errorf("%s", err) - } - km.vIncMut.Unlock() - err = setKeyID(vKeyPrefix, id, o, km.db) - if err != nil { - log.Errorf("%s", err) - } - err = setIDKey(vIDPrefix, id, o, km.db) - if err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label) - setIDLabel(vLabelPrefix, o, lkey, km.db) - return o, lkey -} - -func (km *KeyMap) GetVertexKey(id string) (uint64, bool) { - return getIDKey(vIDPrefix, id, km.db) -} - -//GetVertexID -func (km *KeyMap) GetVertexID(key uint64) (string, bool) { - return getKeyID(vKeyPrefix, key, km.db) -} - -func (km *KeyMap) GetVertexLabel(key uint64) uint64 { - k, _ := getIDLabel(vLabelPrefix, key, km.db) - return k -} - -//GetsertEdgeKey gets or inserts a new uint64 id for a given edge GID string -func (km *KeyMap) GetsertEdgeKey(id, label string) (uint64, uint64) { - o, ok := getIDKey(eIDPrefix, id, km.db) - if !ok { - km.eIncMut.Lock() - o, _ = dbInc(&km.eIncCur, eInc, km.db) - km.eIncMut.Unlock() - if err := setKeyID(eKeyPrefix, id, o, km.db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(eIDPrefix, id, o, km.db); err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label) - if err := setIDLabel(eLabelPrefix, o, lkey, km.db); err != nil { - log.Errorf("%s", err) - } - return o, lkey -} - -//GetEdgeKey gets the uint64 key for a given GID string -func (km *KeyMap) GetEdgeKey(id string) (uint64, bool) { - return getIDKey(eIDPrefix, id, km.db) -} - -//GetEdgeID gets the GID string for a given edge id uint64 -func (km *KeyMap) GetEdgeID(key uint64) (string, bool) { - return getKeyID(eKeyPrefix, key, km.db) -} - -func (km *KeyMap) GetEdgeLabel(key uint64) uint64 { - k, _ := getIDLabel(eLabelPrefix, key, km.db) - return k -} - -//DelVertexKey -func (km *KeyMap) DelVertexKey(id string) error { - key, ok := km.GetVertexKey(id) - if !ok { - return fmt.Errorf("%s vertexKey not found", id) - } - if err := delKeyID(vKeyPrefix, key, km.db); err != nil { - return err - } - if err := delIDKey(vIDPrefix, id, km.db); err != nil { - return err - } - return nil -} - -//DelEdgeKey -func (km *KeyMap) DelEdgeKey(id string) error { - key, ok := km.GetEdgeKey(id) - if !ok { - return fmt.Errorf("%s edgeKey not found", id) - } - if err := delKeyID(eKeyPrefix, key, km.db); err != nil { - return err - } - if err := delIDKey(eIDPrefix, id, km.db); err != nil { - return err - } - return nil -} - -//GetsertLabelKey gets-or-inserts a new label key uint64 for a given string -func (km *KeyMap) GetsertLabelKey(id string) uint64 { - u, ok := getIDKey(lIDPrefix, id, km.db) - if ok { - return u - } - km.lIncMut.Lock() - o, _ := dbInc(&km.lIncCur, lInc, km.db) - km.lIncMut.Unlock() - if err := setKeyID(lKeyPrefix, id, o, km.db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(lIDPrefix, id, o, km.db); err != nil { - log.Errorf("%s", err) - } - return o -} - -func (km *KeyMap) GetLabelKey(id string) (uint64, bool) { - return getIDKey(lIDPrefix, id, km.db) -} - -//GetLabelID gets the GID for a given uint64 label key -func (km *KeyMap) GetLabelID(key uint64) (string, bool) { - return getKeyID(lKeyPrefix, key, km.db) -} - -func getIDKey(prefix []byte, id string, db *pogreb.DB) (uint64, bool) { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - v, err := db.Get(k) - if v == nil || err != nil { - return 0, false - } - key, _ := binary.Uvarint(v) - return key, true -} - -func setIDKey(prefix []byte, id string, key uint64, db *pogreb.DB) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, key) - return db.Put(k, b) -} - -func delIDKey(prefix []byte, id string, db *pogreb.DB) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - return db.Delete(k) -} - -func getIDLabel(prefix byte, key uint64, db *pogreb.DB) (uint64, bool) { - k := make([]byte, 1+binary.MaxVarintLen64) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - v, err := db.Get(k) - if v == nil || err != nil { - return 0, false - } - label, _ := binary.Uvarint(v) - return label, true -} - -func setIDLabel(prefix byte, key uint64, label uint64, db *pogreb.DB) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, label) - - err := db.Put(k, b) - return err -} - -func setKeyID(prefix byte, id string, key uint64, db *pogreb.DB) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Put(k, []byte(id)) -} - -func getKeyID(prefix byte, key uint64, db *pogreb.DB) (string, bool) { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - b, err := db.Get(k) - if b == nil || err != nil { - return "", false - } - return string(b), true -} - -func delKeyID(prefix byte, key uint64, db *pogreb.DB) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Delete(k) -} - -func dbInc(inc *uint64, k []byte, db *pogreb.DB) (uint64, error) { - b := make([]byte, binary.MaxVarintLen64) - if *inc == 0 { - v, _ := db.Get(k) - if v == nil { - binary.PutUvarint(b, incMod) - if err := db.Put(k, b); err != nil { - return 0, err - } - (*inc) += 2 - return 1, nil - } - newInc, _ := binary.Uvarint(v) - *inc = newInc - binary.PutUvarint(b, (*inc)+incMod) - if err := db.Put(k, b); err != nil { - return 0, err - } - o := (*inc) - (*inc)++ - return o, nil - } - o := *inc - (*inc)++ - if *inc%incMod == 0 { - binary.PutUvarint(b, *inc+incMod) - if err := db.Put(k, b); err != nil { - return 0, err - } - } - return o, nil -} diff --git a/grids/old/keys.go b/grids/old/keys.go deleted file mode 100644 index f0724cc1..00000000 --- a/grids/old/keys.go +++ /dev/null @@ -1,166 +0,0 @@ -package grids - -import ( - "encoding/binary" -) - -var vertexPrefix = []byte("v") -var edgePrefix = []byte("e") -var srcEdgePrefix = []byte("s") -var dstEdgePrefix = []byte("d") - -var intSize = 10 - -// VertexKey generates the key given a vertexId -func VertexKey(id uint64) []byte { - out := make([]byte, intSize+1) - out[0] = vertexPrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out -} - -// VertexKeyParse takes a byte array key and returns: -// `graphId`, `vertexId` -func VertexKeyParse(key []byte) uint64 { - id, _ := binary.Uvarint(key[1 : intSize+1]) - return id -} - -// EdgeKey takes the required components of an edge key and returns the byte array -func EdgeKey(id, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], dst) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out -} - -// EdgeKeyPrefix returns the byte array prefix for a particular edge id -func EdgeKeyPrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out -} - -// EdgeKeyParse takes a edge key and returns the elements encoded in it: -// `graph`, `edgeID`, `srcVertexId`, `dstVertexId`, `label` -func EdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - eid, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - did, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} - -// VertexListPrefix returns a byte array prefix for all vertices in a graph -func VertexListPrefix() []byte { - out := make([]byte, 1) - out[0] = vertexPrefix[0] - return out -} - -// EdgeListPrefix returns a byte array prefix for all edges in a graph -func EdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = edgePrefix[0] - return out -} - -// SrcEdgeListPrefix returns a byte array prefix for all entries in the source -// edge index for a graph -func SrcEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = srcEdgePrefix[0] - return out -} - -// DstEdgeListPrefix returns a byte array prefix for all entries in the dest -// edge index for a graph -func DstEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = dstEdgePrefix[0] - return out -} - -// SrcEdgeKey creates a src edge index key -func SrcEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out -} - -// DstEdgeKey creates a dest edge index key -func DstEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out -} - -// SrcEdgeKeyParse takes a src index key entry and parses it into: -// `graph`, `edgeId`, `srcVertexId`, `dstVertexId`, `label`, `etype` -func SrcEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - sid, _ := binary.Uvarint(key[1 : intSize+1]) - did, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} - -// DstEdgeKeyParse takes a dest index key entry and parses it into: -// `graph`, `edgeId`, `dstVertexId`, `srcVertexId`, `label`, `etype` -func DstEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - did, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} - -// SrcEdgeKeyPrefix creates a byte array prefix for a src edge index entry -func SrcEdgeKeyPrefix(eid, src, dst uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out -} - -// DstEdgeKeyPrefix creates a byte array prefix for a dest edge index entry -func DstEdgeKeyPrefix(src, dst, eid uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out -} - -// SrcEdgePrefix returns a byte array prefix for all entries in the source -// edge index a particular vertex (the source vertex) -func SrcEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out -} - -// DstEdgePrefix returns a byte array prefix for all entries in the dest -// edge index a particular vertex (the dest vertex) -func DstEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out -} diff --git a/grids/old/new.go b/grids/old/new.go deleted file mode 100644 index 2d97a7b9..00000000 --- a/grids/old/new.go +++ /dev/null @@ -1,105 +0,0 @@ -package grids - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/cockroachdb/pebble" - - "github.com/akrylysov/pogreb" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/kvi" - "github.com/bmeg/grip/kvi/pebbledb" - "github.com/bmeg/grip/kvindex" - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/timestamp" -) - -// Graph implements the GDB interface using a genertic key/value storage driver -type Graph struct { - graphID string - graphKey uint64 - - keyMap *KeyMap - keykv pogreb.DB - - graphkv kvi.KVInterface // *pebble.DB - indexkv kvi.KVInterface // *pebble.DB - idx *kvindex.KVIndex - ts *timestamp.Timestamp -} - -// Close the connection -func (g *Graph) Close() error { - g.keyMap.Close() - g.graphkv.Close() - g.indexkv.Close() - return nil -} - -// AddGraph creates a new graph named `graph` -func (kgraph *GDB) AddGraph(graph string) error { - err := gripql.ValidateGraphName(graph) - if err != nil { - return err - } - g, err := newGraph(kgraph.basePath, graph) - if err != nil { - return err - } - kgraph.drivers[graph] = g - return nil -} - -func newGraph(baseDir, name string) (*Graph, error) { - dbPath := filepath.Join(baseDir, name) - - log.Infof("Creating new GRIDS graph %s", name) - - _, err := os.Stat(dbPath) - if os.IsNotExist(err) { - os.Mkdir(dbPath, 0700) - } - keykvPath := fmt.Sprintf("%s/keymap", dbPath) - graphkvPath := fmt.Sprintf("%s/graph", dbPath) - indexkvPath := fmt.Sprintf("%s/index", dbPath) - keykv, err := pogreb.Open(keykvPath, nil) - if err != nil { - return nil, err - } - graphkv, err := pebble.Open(graphkvPath, &pebble.Options{}) - if err != nil { - return nil, err - } - indexkv, err := pebble.Open(indexkvPath, &pebble.Options{}) - if err != nil { - return nil, err - } - ts := timestamp.NewTimestamp() - - o := &Graph{ - keyMap: NewKeyMap(keykv), - graphkv: pebbledb.WrapPebble(graphkv), - indexkv: pebbledb.WrapPebble(indexkv), - ts: &ts, - idx: kvindex.NewIndex(pebbledb.WrapPebble(indexkv)), - } - - return o, nil -} - -// DeleteGraph deletes `graph` -func (kgraph *GDB) DeleteGraph(graph string) error { - err := gripql.ValidateGraphName(graph) - if err != nil { - return nil - } - if d, ok := kgraph.drivers[graph]; ok { - d.Close() - delete(kgraph.drivers, graph) - } - dbPath := filepath.Join(kgraph.basePath, graph) - os.RemoveAll(dbPath) - return nil -} diff --git a/grids/old/raw_graph.go b/grids/old/raw_graph.go deleted file mode 100644 index 96efd565..00000000 --- a/grids/old/raw_graph.go +++ /dev/null @@ -1,262 +0,0 @@ -package grids - -import ( - "bytes" - "context" - - "github.com/bmeg/grip/kvi" - "github.com/bmeg/grip/util/setcmp" -) - -func (ggraph *Graph) RawGetVertexList(ctx context.Context) <-chan *GRIDDataElement { - o := make(chan *GRIDDataElement, 100) - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - vPrefix := VertexListPrefix() - for it.Seek(vPrefix); it.Valid() && bytes.HasPrefix(it.Key(), vPrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - keyValue := it.Key() - vkey := VertexKeyParse(keyValue) - lkey := ggraph.keyMap.GetVertexLabel(vkey) - o <- &GRIDDataElement{ - Gid: vkey, - Label: lkey, - Data: map[string]interface{}{}, - Loaded: false, - } - } - return nil - }) - }() - return o -} - -func (ggraph *Graph) RawGetEdgeList(ctx context.Context) <-chan *GRIDDataElement { - o := make(chan *GRIDDataElement, 100) - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - ePrefix := EdgeListPrefix() - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - keyValue := it.Key() - ekey, srcvkey, dstvkey, lkey := EdgeKeyParse(keyValue) - o <- &GRIDDataElement{ - Gid: ekey, - Label: lkey, - From: srcvkey, - To: dstvkey, - Data: map[string]interface{}{}, - Loaded: false, - } - } - return nil - }) - }() - return o -} - -func (ggraph *Graph) RawGetVertexChannel(reqChan chan *RawElementLookup) <-chan *RawElementLookup { - o := make(chan *RawElementLookup, 100) - go func() { - defer close(o) - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - vkey := req.ID - lkey := ggraph.keyMap.GetVertexLabel(vkey) - o <- &RawElementLookup{ - Element: &GRIDDataElement{ - Gid: vkey, - Label: lkey, - Data: map[string]interface{}{}, - Loaded: false, - }, - ID: req.ID, - Ref: req.Ref, - } - } - } - }() - return o -} - -func (ggraph *Graph) RawGetOutChannel(reqChan chan *RawElementLookup, edgeLabels []string) chan *RawElementLookup { - o := make(chan *RawElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - ePrefix := SrcEdgePrefix(req.ID) - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - keyValue := it.Key() - _, _, dstvkey, lkey := SrcEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || setcmp.ContainsUint(edgeLabelKeys, lkey) { - dstlkey := ggraph.keyMap.GetVertexLabel(dstvkey) - o <- &RawElementLookup{ - Element: &GRIDDataElement{ - Gid: dstvkey, - Label: dstlkey, - Data: map[string]interface{}{}, - Loaded: false, - }, - ID: req.ID, - Ref: req.Ref, - } - } - } - } - } - return nil - }) - }() - return o -} - -func (ggraph *Graph) RawGetInChannel(reqChan chan *RawElementLookup, edgeLabels []string) chan *RawElementLookup { - o := make(chan *RawElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - ePrefix := DstEdgePrefix(req.ID) - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - keyValue := it.Key() - _, srcvkey, _, lkey := DstEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || setcmp.ContainsUint(edgeLabelKeys, lkey) { - srclkey := ggraph.keyMap.GetVertexLabel(srcvkey) - o <- &RawElementLookup{ - Element: &GRIDDataElement{ - Gid: srcvkey, - Label: srclkey, - Data: map[string]interface{}{}, - Loaded: false, - }, - ID: req.ID, - Ref: req.Ref, - } - } - } - } - } - return nil - }) - }() - return o -} - -func (ggraph *Graph) RawGetOutEdgeChannel(reqChan chan *RawElementLookup, edgeLabels []string) chan *RawElementLookup { - o := make(chan *RawElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - ePrefix := SrcEdgePrefix(req.ID) - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - keyValue := it.Key() - ekey, srcvkey, dstvkey, lkey := SrcEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || setcmp.ContainsUint(edgeLabelKeys, lkey) { - o <- &RawElementLookup{ - Element: &GRIDDataElement{ - Gid: ekey, - Label: lkey, - From: srcvkey, - To: dstvkey, - Data: map[string]interface{}{}, - Loaded: false, - }, - ID: req.ID, - Ref: req.Ref, - } - } - } - } - } - return nil - }) - }() - return o -} - -func (ggraph *Graph) RawGetInEdgeChannel(reqChan chan *RawElementLookup, edgeLabels []string) chan *RawElementLookup { - o := make(chan *RawElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i]) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } - go func() { - defer close(o) - ggraph.graphkv.View(func(it kvi.KVIterator) error { - for req := range reqChan { - if req.IsSignal() { - o <- req - } else { - ePrefix := DstEdgePrefix(req.ID) - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - keyValue := it.Key() - ekey, srcvkey, dstvkey, lkey := DstEdgeKeyParse(keyValue) - if len(edgeLabels) == 0 || setcmp.ContainsUint(edgeLabelKeys, lkey) { - o <- &RawElementLookup{ - Element: &GRIDDataElement{ - Gid: ekey, - Label: lkey, - From: srcvkey, - To: dstvkey, - Data: map[string]interface{}{}, - Loaded: false, - }, - ID: req.ID, - Ref: req.Ref, - } - } - } - } - } - return nil - }) - }() - return o -} diff --git a/grids/old/raw_processors.go b/grids/old/raw_processors.go deleted file mode 100644 index 21c5e519..00000000 --- a/grids/old/raw_processors.go +++ /dev/null @@ -1,301 +0,0 @@ -package grids - -import ( - "context" - - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/util/setcmp" -) - -// ElementLookup request to look up data -type RawElementLookup struct { - ID uint64 - Ref *GRIDTraveler - Element *GRIDDataElement -} - -func (rel *RawElementLookup) IsSignal() bool { - if rel.Ref != nil { - return rel.Ref.IsSignal() - } - return false -} - -type PathVProc struct { - db *Graph - ids []string -} - -func (r *PathVProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - go func() { - for range in { - } - defer close(out) - if len(r.ids) == 0 { - for elem := range r.db.RawGetVertexList(ctx) { - out <- &GRIDTraveler{Current: elem, Graph: r.db} - } - } else { - for _, i := range r.ids { - if key, ok := r.db.keyMap.GetVertexKey(i); ok { - label := r.db.keyMap.GetVertexLabel(key) - o := &GRIDTraveler{ - Current: &GRIDDataElement{Gid: key, Label: label, Data: map[string]interface{}{}, Loaded: false}, - Graph: r.db, - } - out <- o - } - } - } - }() - return ctx -} - -type PathOutProc struct { - db *Graph - labels []string -} - -func (r *PathOutProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, true, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.Gid, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetOutChannel(queryChan, r.labels) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -type PathInProc struct { - db *Graph - labels []string -} - -func (r *PathInProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, true, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.Gid, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetInChannel(queryChan, r.labels) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -type PathOutEProc struct { - db *Graph - labels []string -} - -func (r *PathOutEProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, false, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.Gid, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetOutEdgeChannel(queryChan, r.labels) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -// PathOutAdjEProc process edge to out -type PathOutEdgeAdjProc struct { - db *Graph -} - -func (r *PathOutEdgeAdjProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, true, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.To, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetVertexChannel(queryChan) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -type PathInEProc struct { - db *Graph - labels []string -} - -func (r *PathInEProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, true, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.Gid, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetInEdgeChannel(queryChan, r.labels) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -type PathInEdgeAdjProc struct { - db *Graph - labels []string -} - -func (r *PathInEdgeAdjProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan *RawElementLookup, 100) - go func() { - defer close(queryChan) - for i := range in { - gi := NewGRIDTraveler(i, true, r.db) - if gi.IsSignal() { - queryChan <- &RawElementLookup{ - Ref: gi, - } - } else { - queryChan <- &RawElementLookup{ - ID: gi.Current.From, - Ref: gi, - } - } - } - }() - go func() { - defer close(out) - for ov := range r.db.RawGetVertexChannel(queryChan) { - if ov.IsSignal() { - out <- ov.Ref - } else { - i := ov.Ref - out <- i.AddRawCurrent(ov.Element) - } - } - }() - return ctx -} - -type PathLabelProc struct { - db *Graph - labels []string -} - -func (r *PathLabelProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - labels := []uint64{} - for i := range r.labels { - if j, ok := r.db.keyMap.GetLabelKey(r.labels[i]); ok { - labels = append(labels, j) - } - } - go func() { - defer close(out) - for i := range in { - if i.IsSignal() { - out <- i - } - gi := NewGRIDTraveler(i, true, r.db) - if setcmp.ContainsUint(labels, gi.Current.Label) { - out <- i - } - } - }() - return ctx -} diff --git a/grids/old/schema.go b/grids/old/schema.go deleted file mode 100644 index 21d3e343..00000000 --- a/grids/old/schema.go +++ /dev/null @@ -1,91 +0,0 @@ -package grids - -import ( - "context" - "fmt" - - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/util" - "google.golang.org/protobuf/types/known/structpb" -) - -// BuildSchema returns the schema of a specific graph in the database -func (ma *GDB) BuildSchema(ctx context.Context, graph string, sampleN uint32, random bool) (*gripql.Graph, error) { - var vSchema []*gripql.Vertex - var eSchema []*gripql.Edge - var err error - - log.WithFields(log.Fields{"graph": graph}).Debug("Starting KV GetSchema call") - - if g, ok := ma.drivers[graph]; ok { - vSchema, eSchema, err = g.sampleSchema(ctx, sampleN, random) - if err != nil { - return nil, fmt.Errorf("getting vertex schema: %v", err) - } - - schema := &gripql.Graph{Graph: graph, Vertices: vSchema, Edges: eSchema} - log.WithFields(log.Fields{"graph": graph}).Debug("Finished GetSchema call") - return schema, nil - - } - return nil, fmt.Errorf("Graph not found") -} - -func (gi *Graph) sampleSchema(ctx context.Context, n uint32, random bool) ([]*gripql.Vertex, []*gripql.Edge, error) { - labelField := fmt.Sprintf("v.label") - labels := []string{} - for i := range gi.idx.FieldTerms(labelField) { - labels = append(labels, i.(string)) - } - - vOutput := []*gripql.Vertex{} - eOutput := []*gripql.Edge{} - fromToPairs := make(fromto) - - for _, label := range labels { - schema := map[string]interface{}{} - for i := range gi.idx.GetTermMatch(context.Background(), labelField, label, int(n)) { - v := gi.GetVertex(i, true) - data := v.Data - ds := gripql.GetDataFieldTypes(data) - util.MergeMaps(schema, ds) - - reqChan := make(chan gdbi.ElementLookup, 1) - reqChan <- gdbi.ElementLookup{ID: i} - close(reqChan) - for e := range gi.GetOutEdgeChannel(ctx, reqChan, true, false, []string{}) { - o := gi.GetVertex(e.Edge.To, false) - k := fromtokey{from: v.Label, to: o.Label, label: e.Edge.Label} - ds := gripql.GetDataFieldTypes(e.Edge.Data) - if p, ok := fromToPairs[k]; ok { - fromToPairs[k] = util.MergeMaps(p, ds) - } else { - fromToPairs[k] = ds - } - } - } - sSchema, _ := structpb.NewStruct(schema) - vSchema := &gripql.Vertex{Gid: label, Label: label, Data: sSchema} - vOutput = append(vOutput, vSchema) - } - for k, v := range fromToPairs { - sV, _ := structpb.NewStruct(v.(map[string]interface{})) - eSchema := &gripql.Edge{ - Gid: fmt.Sprintf("(%s)--%s->(%s)", k.from, k.label, k.to), - Label: k.label, - From: k.from, - To: k.to, - Data: sV, - } - eOutput = append(eOutput, eSchema) - } - return vOutput, eOutput, nil -} - -type fromtokey struct { - from, to, label string -} - -type fromto map[fromtokey]interface{} diff --git a/grids/old/statement_compiler.go b/grids/old/statement_compiler.go deleted file mode 100644 index 189f8fef..00000000 --- a/grids/old/statement_compiler.go +++ /dev/null @@ -1,250 +0,0 @@ -package grids - -import ( - "fmt" - - "github.com/bmeg/grip/engine/logic" - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/travelerpath" - "github.com/bmeg/grip/util/protoutil" -) - -type GridStmtCompiler struct { - db Graph -} - -func (sc *GridStmtCompiler) V(stmt *gripql.GraphStatement_V, ps *gdbi.State) (gdbi.Processor, error) { - ids := protoutil.AsStringList(stmt.V) - return &LookupVerts{db: sc.db, ids: ids, loadData: ps.StepLoadData()}, nil -} - -func (sc *GridStmtCompiler) E(stmt *gripql.GraphStatement_E, ps *gdbi.State) (gdbi.Processor, error) { - ids := protoutil.AsStringList(stmt.E) - return &LookupEdges{db: sc.db, ids: ids, loadData: ps.StepLoadData()}, nil -} - -func (sc *GridStmtCompiler) In(stmt *gripql.GraphStatement_In, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.In) - if ps.LastType == gdbi.VertexData { - return &LookupVertexAdjIn{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else if ps.LastType == gdbi.EdgeData { - return &LookupEdgeAdjIn{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else { - return nil, fmt.Errorf(`"in" statement is only valid for edge or vertex types not: %s`, ps.LastType.String()) - } -} - -func (sc *GridStmtCompiler) InNull(stmt *gripql.GraphStatement_InNull, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.InNull) - if ps.LastType == gdbi.VertexData { - return &LookupVertexAdjIn{db: sc.db, labels: labels, loadData: ps.StepLoadData(), emitNull: true}, nil - } else if ps.LastType == gdbi.EdgeData { - return &LookupEdgeAdjIn{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else { - return nil, fmt.Errorf(`"in" statement is only valid for edge or vertex types not: %s`, ps.LastType.String()) - } -} -func (sc *GridStmtCompiler) Out(stmt *gripql.GraphStatement_Out, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.Out) - if ps.LastType == gdbi.VertexData { - return &LookupVertexAdjOut{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else if ps.LastType == gdbi.EdgeData { - return &LookupEdgeAdjOut{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else { - return nil, fmt.Errorf(`"out" statement is only valid for edge or vertex types not: %s`, ps.LastType.String()) - } -} - -func (sc *GridStmtCompiler) OutNull(stmt *gripql.GraphStatement_OutNull, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.OutNull) - if ps.LastType == gdbi.VertexData { - return &LookupVertexAdjOut{db: sc.db, labels: labels, loadData: ps.StepLoadData(), emitNull: true}, nil - } else if ps.LastType == gdbi.EdgeData { - return &LookupEdgeAdjOut{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil - } else { - return nil, fmt.Errorf(`"out" statement is only valid for edge or vertex types not: %s`, ps.LastType.String()) - } -} - -func (sc *GridStmtCompiler) Both(stmt *gripql.GraphStatement_Both, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.Both) - if ps.LastType == gdbi.VertexData { - return &both{db: sc.db, labels: labels, lastType: gdbi.VertexData, toType: gdbi.VertexData, loadData: ps.StepLoadData()}, nil - } else if ps.LastType == gdbi.EdgeData { - return &both{db: sc.db, labels: labels, lastType: gdbi.EdgeData, toType: gdbi.VertexData, loadData: ps.StepLoadData()}, nil - } else { - return nil, fmt.Errorf(`"both" statement is only valid for edge or vertex types not: %s`, ps.LastType.String()) - } -} - -func (sc *GridStmtCompiler) InE(stmt *gripql.GraphStatement_InE, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.InE) - return &InE{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil -} - -func (sc *GridStmtCompiler) InENull(stmt *gripql.GraphStatement_InENull, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.InENull) - ps.LastType = gdbi.EdgeData - return &InE{db: sc.db, labels: labels, loadData: ps.StepLoadData(), emitNull: true}, nil -} - -func (sc *GridStmtCompiler) OutE(stmt *gripql.GraphStatement_OutE, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.OutE) - ps.LastType = gdbi.EdgeData - return &OutE{db: sc.db, labels: labels, loadData: ps.StepLoadData()}, nil -} - -func (sc *GridStmtCompiler) OutENull(stmt *gripql.GraphStatement_OutENull, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.OutENull) - ps.LastType = gdbi.EdgeData - return &OutE{db: sc.db, labels: labels, loadData: ps.StepLoadData(), emitNull: true}, nil -} - -func (sc *GridStmtCompiler) BothE(stmt *gripql.GraphStatement_BothE, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.BothE) - ps.LastType = gdbi.EdgeData - return &both{db: sc.db, labels: labels, lastType: gdbi.VertexData, toType: gdbi.EdgeData, loadData: ps.StepLoadData()}, nil -} - -func (sc *GridStmtCompiler) Has(stmt *gripql.GraphStatement_Has, ps *gdbi.State) (gdbi.Processor, error) { - return &Has{stmt.Has}, nil -} - -func (sc *GridStmtCompiler) HasLabel(stmt *gripql.GraphStatement_HasLabel, ps *gdbi.State) (gdbi.Processor, error) { - labels := protoutil.AsStringList(stmt.HasLabel) - if len(labels) == 0 { - return nil, fmt.Errorf(`no labels provided to "HasLabel" statement`) - } - return &HasLabel{labels}, nil -} - -func (sc *GridStmtCompiler) HasKey(stmt *gripql.GraphStatement_HasKey, ps *gdbi.State) (gdbi.Processor, error) { - keys := protoutil.AsStringList(stmt.HasKey) - if len(keys) == 0 { - return nil, fmt.Errorf(`no keys provided to "HasKey" statement`) - } - return &HasKey{keys}, nil -} - -func (sc *GridStmtCompiler) HasID(stmt *gripql.GraphStatement_HasId, ps *gdbi.State) (gdbi.Processor, error) { - ids := protoutil.AsStringList(stmt.HasId) - if len(ids) == 0 { - return nil, fmt.Errorf(`no ids provided to "HasId" statement`) - } - return &HasID{ids}, nil -} - -func (sc *GridStmtCompiler) Limit(stmt *gripql.GraphStatement_Limit, ps *gdbi.State) (gdbi.Processor, error) { - return &Limit{stmt.Limit}, nil -} - -func (sc *GridStmtCompiler) Skip(stmt *gripql.GraphStatement_Skip, ps *gdbi.State) (gdbi.Processor, error) { - return &Skip{stmt.Skip}, nil -} - -func (sc *GridStmtCompiler) Range(stmt *gripql.GraphStatement_Range, ps *gdbi.State) (gdbi.Processor, error) { - return &Range{start: stmt.Range.Start, stop: stmt.Range.Stop}, nil -} - -func (sc *GridStmtCompiler) Count(stmt *gripql.GraphStatement_Count, ps *gdbi.State) (gdbi.Processor, error) { - return &Count{}, nil -} - -func (sc *GridStmtCompiler) Distinct(stmt *gripql.GraphStatement_Distinct, ps *gdbi.State) (gdbi.Processor, error) { - fields := protoutil.AsStringList(stmt.Distinct) - if len(fields) == 0 { - fields = append(fields, "_gid") - } - return &Distinct{fields}, nil -} - -func (sc *GridStmtCompiler) As(stmt *gripql.GraphStatement_As, ps *gdbi.State) (gdbi.Processor, error) { - if stmt.As == "" { - return nil, fmt.Errorf(`"mark" statement cannot have an empty name`) - } - if err := gripql.ValidateFieldName(stmt.As); err != nil { - return nil, fmt.Errorf(`"mark" statement invalid; %v`, err) - } - if stmt.As == travelerpath.Current { - return nil, fmt.Errorf(`"mark" statement invalid; uses reserved name %s`, travelerpath.Current) - } - ps.MarkTypes[stmt.As] = ps.LastType - return &Marker{stmt.As}, nil -} - -func (sc *GridStmtCompiler) Set(stmt *gripql.GraphStatement_Set, ps *gdbi.State) (gdbi.Processor, error) { - return &ValueSet{key: stmt.Set.Key, value: stmt.Set.Value.AsInterface()}, nil -} - -func (sc *GridStmtCompiler) Increment(stmt *gripql.GraphStatement_Increment, ps *gdbi.State) (gdbi.Processor, error) { - return &ValueIncrement{key: stmt.Increment.Key, value: stmt.Increment.Value}, nil -} - -func (sc *GridStmtCompiler) Mark(stmt *gripql.GraphStatement_Mark, ps *gdbi.State) (gdbi.Processor, error) { - return &logic.JumpMark{Name: stmt.Mark}, nil -} - -func (sc *GridStmtCompiler) Jump(stmt *gripql.GraphStatement_Jump, ps *gdbi.State) (gdbi.Processor, error) { - return &logic.Jump{Mark: stmt.Jump.Mark, Stmt: stmt.Jump.Expression, Emit: stmt.Jump.Emit}, nil -} - -func (sc *GridStmtCompiler) Select(stmt *gripql.GraphStatement_Select, ps *gdbi.State) (gdbi.Processor, error) { - switch len(stmt.Select.Marks) { - case 0: - return nil, fmt.Errorf(`"select" statement has an empty list of mark names`) - case 1: - ps.LastType = ps.MarkTypes[stmt.Select.Marks[0]] - return &MarkSelect{stmt.Select.Marks[0]}, nil - default: - ps.LastType = gdbi.SelectionData - return &Selector{stmt.Select.Marks}, nil - } -} - -func (sc *GridStmtCompiler) Render(stmt *gripql.GraphStatement_Render, ps *gdbi.State) (gdbi.Processor, error) { - return &Render{stmt.Render.AsInterface()}, nil -} - -func (sc *GridStmtCompiler) Path(stmt *gripql.GraphStatement_Path, ps *gdbi.State) (gdbi.Processor, error) { - return &Path{stmt.Path.AsSlice()}, nil -} - -func (sc *GridStmtCompiler) Unwind(stmt *gripql.GraphStatement_Unwind, ps *gdbi.State) (gdbi.Processor, error) { - return &Unwind{stmt.Unwind}, nil -} - -func (sc *GridStmtCompiler) Fields(stmt *gripql.GraphStatement_Fields, ps *gdbi.State) (gdbi.Processor, error) { - fields := protoutil.AsStringList(stmt.Fields) - return &Fields{fields}, nil -} - -func (sc *GridStmtCompiler) Aggregate(stmt *gripql.GraphStatement_Aggregate, ps *gdbi.State) (gdbi.Processor, error) { - aggs := make(map[string]interface{}) - for _, a := range stmt.Aggregate.Aggregations { - if _, ok := aggs[a.Name]; ok { - return nil, fmt.Errorf("duplicate aggregation name '%s' found; all aggregations must have a unique name", a.Name) - } - } - return &aggregate{stmt.Aggregate.Aggregations}, nil -} - -func (sc *GridStmtCompiler) Custom(gs *gripql.GraphStatement, ps *gdbi.State) (gdbi.Processor, error) { - - switch stmt := gs.GetStatement().(type) { - - //Custom graph statements - case *gripql.GraphStatement_LookupVertsIndex: - ps.LastType = gdbi.VertexData - return &LookupVertsIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil - - case *gripql.GraphStatement_EngineCustom: - proc := stmt.Custom.(gdbi.CustomProcGen) - ps.LastType = proc.GetType() - return proc.GetProcessor(sc.db, ps) - - default: - return nil, fmt.Errorf("grip compile: unknown statement type: %s", gs.GetStatement()) - } - -} diff --git a/grids/old/traveler.go b/grids/old/traveler.go deleted file mode 100644 index 476b3f99..00000000 --- a/grids/old/traveler.go +++ /dev/null @@ -1,269 +0,0 @@ -package grids - -import ( - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/util/copy" -) - -type GRIDTraveler struct { - Graph *Graph - Current *GRIDDataElement - Marks map[string]*GRIDDataElement - Selections map[string]*GRIDDataElement - Aggregation *gdbi.Aggregate - Count uint32 - Render interface{} - Path []GRIDDataElementID - Signal *gdbi.Signal -} - -type GRIDDataElementID struct { - IsVertex bool - Gid uint64 -} - -type GRIDDataElement struct { - Gid uint64 - To uint64 - From uint64 - Label uint64 - Data map[string]interface{} - Loaded bool -} - -func (rd *GRIDDataElement) VertexDataElement(ggraph *Graph) *gdbi.Vertex { - Gid, _ := ggraph.keyMap.GetVertexID(rd.Gid) - Label, _ := ggraph.keyMap.GetLabelID(rd.Label) - return &gdbi.DataElement{ID: Gid, Label: Label, Data: rd.Data, Loaded: rd.Loaded} -} - -func (rd *GRIDDataElement) EdgeDataElement(ggraph *Graph) *gdbi.Edge { - Gid, _ := ggraph.keyMap.GetEdgeID(rd.Gid) - Label, _ := ggraph.keyMap.GetLabelID(rd.Label) - To, _ := ggraph.keyMap.GetVertexID(rd.To) - From, _ := ggraph.keyMap.GetVertexID(rd.From) - return &gdbi.DataElement{ID: Gid, To: To, From: From, Label: Label, Data: rd.Data, Loaded: rd.Loaded} -} - -func (rd *GRIDDataElement) DataElement(ggraph *Graph) *gdbi.DataElement { - if rd.To != 0 { - return rd.EdgeDataElement(ggraph) - } - return rd.VertexDataElement(ggraph) -} - -func (rd *GRIDDataElement) Copy() *GRIDDataElement { - return &GRIDDataElement{ - Gid: rd.Gid, - Label: rd.Label, - To: rd.To, - From: rd.From, - Loaded: rd.Loaded, - Data: copy.DeepCopy(rd.Data).(map[string]interface{}), - } -} - -func DataElementToGRID(d *gdbi.DataElement, g *Graph) (*GRIDDataElement, error) { - if d.To != "" { - Gid, _ := g.keyMap.GetEdgeKey(d.ID) - Label, _ := g.keyMap.GetLabelKey(d.Label) - To, _ := g.keyMap.GetVertexKey(d.To) - From, _ := g.keyMap.GetVertexKey(d.From) - return &GRIDDataElement{ - Gid: Gid, Label: Label, To: To, From: From, Data: d.Data, Loaded: d.Loaded, - }, nil - } - Gid, _ := g.keyMap.GetVertexKey(d.ID) - Label, _ := g.keyMap.GetLabelKey(d.Label) - o := &GRIDDataElement{ - Gid: Gid, Label: Label, Data: d.Data, Loaded: d.Loaded, - } - return o, nil -} - -func NewGRIDTraveler(tr gdbi.Traveler, isVertex bool, gg *Graph) *GRIDTraveler { - if a, ok := tr.(*GRIDTraveler); ok { - return a - } - - if tr.IsSignal() { - s := tr.GetSignal() - return &GRIDTraveler{Signal: &s} - } - - cur := tr.GetCurrent() - el, _ := DataElementToGRID(cur, gg) - o := &GRIDTraveler{ - Graph: gg, - Current: el, - Aggregation: tr.GetAggregation(), - Count: tr.GetCount(), - Render: tr.GetRender(), - Path: []GRIDDataElementID{}, - Marks: map[string]*GRIDDataElement{}, - } - for _, e := range tr.GetPath() { - if e.Vertex != "" { - l, _ := gg.keyMap.GetVertexKey(e.Vertex) - o.Path = append(o.Path, GRIDDataElementID{IsVertex: true, Gid: l}) - } else { - l, _ := gg.keyMap.GetVertexKey(e.Edge) - o.Path = append(o.Path, GRIDDataElementID{IsVertex: false, Gid: l}) - } - } - for _, k := range tr.ListMarks() { - m := tr.GetMark(k) - o.Marks[k], _ = DataElementToGRID(m, gg) - } - - return o -} - -func (tr *GRIDTraveler) GetSignal() gdbi.Signal { - if tr.Signal == nil { - return gdbi.Signal{} - } - return *tr.Signal -} - -func (tr *GRIDTraveler) IsSignal() bool { - return tr.Signal != nil -} - -func (tr *GRIDTraveler) IsNull() bool { - return tr.Current != nil -} - -// AddCurrent creates a new copy of the travel with new 'current' value -func (t *GRIDTraveler) AddCurrent(r gdbi.DataRef) gdbi.Traveler { - a := t.GRIDCopy() - c, _ := DataElementToGRID(r, t.Graph) - a.Current = c - return a -} - -func (t *GRIDTraveler) AddRawCurrent(r *GRIDDataElement) *GRIDTraveler { - o := GRIDTraveler{ - Graph: t.Graph, - Marks: map[string]*GRIDDataElement{}, - Path: make([]GRIDDataElementID, len(t.GetPath())+1), - } - - for k, v := range t.Marks { - o.Marks[k] = v.Copy() - } - for i := range t.Path { - o.Path[i] = t.Path[i] - } - if r == nil { - o.Path[len(t.Path)] = GRIDDataElementID{} - } else if r.To != 0 { - o.Path[len(t.Path)] = GRIDDataElementID{Gid: r.Gid, IsVertex: false} - } else { - o.Path[len(t.Path)] = GRIDDataElementID{Gid: r.Gid, IsVertex: true} - } - o.Current = r - return &o -} - -func (t *GRIDTraveler) GetCurrentID() string { - if t.Current.To == 0 { - s, _ := t.Graph.keyMap.GetVertexID(t.Current.Gid) - return s - } else { - s, _ := t.Graph.keyMap.GetEdgeID(t.Current.Gid) - return s - } -} - -// AddCurrent creates a new copy of the travel with new 'current' value -func (t *GRIDTraveler) Copy() gdbi.Traveler { - return t.GRIDCopy() -} - -func (t *GRIDTraveler) GRIDCopy() *GRIDTraveler { - o := GRIDTraveler{ - Graph: t.Graph, - Marks: map[string]*GRIDDataElement{}, - Path: make([]GRIDDataElementID, len(t.GetPath())), - Signal: t.Signal, - Count: t.Count, - Aggregation: t.Aggregation, - } - for k, v := range t.Marks { - o.Marks[k] = v.Copy() - } - for i := range t.Path { - o.Path[i] = t.Path[i] - } - o.Current = t.Current - return &o -} - -// HasMark checks to see if a results is stored in a travelers statemap -func (t *GRIDTraveler) HasMark(label string) bool { - _, ok := t.Marks[label] - return ok -} - -// ListMarks returns the list of marks in a travelers statemap -func (t *GRIDTraveler) ListMarks() []string { - marks := []string{} - for k := range t.Marks { - marks = append(marks, k) - } - return marks -} - -// AddMark adds a result to travels state map using `label` as the name -func (t *GRIDTraveler) AddMark(label string, r gdbi.DataRef) gdbi.Traveler { - o := t.GRIDCopy() - n, _ := DataElementToGRID(r, t.Graph) - o.Marks[label] = n - return o -} - -// GetMark gets stored result in travels state using its label -func (t *GRIDTraveler) GetMark(label string) *gdbi.DataElement { - return t.Marks[label].DataElement(t.Graph) -} - -// GetCurrent get current result value attached to the traveler -func (t *GRIDTraveler) GetCurrent() *gdbi.DataElement { - return t.Current.DataElement(t.Graph) -} - -func (t *GRIDTraveler) GetCount() uint32 { - return t.Count -} - -func (t *GRIDTraveler) GetSelections() map[string]*gdbi.DataElement { - o := map[string]*gdbi.DataElement{} - for k, v := range t.Selections { - o[k] = v.DataElement(t.Graph) - } - return o -} - -func (t *GRIDTraveler) GetRender() interface{} { - return t.Render -} - -func (t *GRIDTraveler) GetPath() []gdbi.DataElementID { - out := make([]gdbi.DataElementID, len(t.Path)) - for i := range t.Path { - e := t.Path[i] - if e.IsVertex { - s, _ := t.Graph.keyMap.GetVertexID(e.Gid) - out[i] = gdbi.DataElementID{Vertex: s} - } else { - s, _ := t.Graph.keyMap.GetEdgeID(e.Gid) - out[i] = gdbi.DataElementID{Edge: s} - } - } - return out -} - -func (t GRIDTraveler) GetAggregation() *gdbi.Aggregate { - return t.Aggregation -} diff --git a/server/server.go b/server/server.go index d0c183a8..d9eba11a 100644 --- a/server/server.go +++ b/server/server.go @@ -238,7 +238,7 @@ func (server *GripServer) Serve(pctx context.Context) error { prefix := fmt.Sprintf("/%s/", name) mux.Handle(prefix, http.StripPrefix(prefix, handler)) } else { - log.Errorf("Unable to load plugin %s", name) + log.Errorf("Unable to load plugin %s: %s", name, err) } }