From 64cc080b8f5c2e7c3ee80f51be8d193e1e641821 Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 09:28:02 -0700 Subject: [PATCH 1/9] Experimental NATS only implementation --- go.mod | 47 +-- go.sum | 465 +------------------------ lib/control_pane.go | 184 ---------- lib/log_db_script.sql | 14 - lib/raft.go | 367 -------------------- lib/replication_event.go | 16 + lib/replicator.go | 107 ++++++ lib/sqlite_log_db.go | 676 ------------------------------------ lib/sqlite_state_machine.go | 387 --------------------- marmot.go | 76 ++-- 10 files changed, 170 insertions(+), 2169 deletions(-) delete mode 100644 lib/control_pane.go delete mode 100644 lib/log_db_script.sql delete mode 100644 lib/raft.go create mode 100644 lib/replication_event.go create mode 100644 lib/replicator.go delete mode 100644 lib/sqlite_log_db.go delete mode 100644 lib/sqlite_state_machine.go diff --git a/go.mod b/go.mod index 8cb72db..505998b 100644 --- a/go.mod +++ b/go.mod @@ -7,55 +7,20 @@ require ( github.com/doug-martin/goqu/v9 v9.18.0 github.com/fsnotify/fsnotify v1.5.4 github.com/fxamacker/cbor/v2 v2.4.0 - github.com/gin-gonic/gin v1.4.0 - github.com/lni/dragonboat/v3 v3.3.5 github.com/mattn/go-sqlite3 v1.14.15 + github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0 github.com/rs/zerolog v1.27.0 github.com/samber/lo v1.27.0 ) require ( - github.com/VictoriaMetrics/metrics v1.6.2 // indirect - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/cockroachdb/errors v1.9.0 // indirect - github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect - github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect - github.com/cockroachdb/redact v1.1.3 // indirect - github.com/getsentry/sentry-go v0.12.0 // indirect - github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 // indirect - github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect - github.com/hashicorp/go-msgpack v0.5.3 // indirect - github.com/hashicorp/go-multierror v1.0.0 // indirect - github.com/hashicorp/go-sockaddr v1.0.0 // indirect - github.com/hashicorp/golang-lru v0.5.0 // indirect - github.com/hashicorp/memberlist v0.2.2 // indirect - github.com/json-iterator/go v1.1.9 // indirect - github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/lni/goutils v1.3.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/miekg/dns v1.1.26 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect - github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - github.com/ugorji/go/codec v1.1.7 // indirect - github.com/valyala/fastrand v1.0.0 // indirect - github.com/valyala/histogram v1.0.1 // indirect + github.com/nats-io/nats-server/v2 v2.9.1 // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect - golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/go-playground/validator.v8 v8.18.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect ) diff --git a/go.sum b/go.sum index 7e3057e..80044aa 100644 --- a/go.sum +++ b/go.sum @@ -1,514 +1,85 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= -github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/VictoriaMetrics/metrics v1.6.2 h1:VMe8c8ZBPgNVZkPoT06LsoU2nb+8e7iPaOWbVRNhxjo= -github.com/VictoriaMetrics/metrics v1.6.2/go.mod h1:LU2j9qq7xqZYXz8tF3/RQnB2z2MbZms5TDiIg9/NHiQ= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.7.5/go.mod h1:m/IWRCPXYZ6TvLLDuC0kfLR1pp/+BiZ0h16WHaBMRMM= -github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= -github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= -github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= -github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb h1:dqFirML/6RMDwkge7Tqf33qE0ORbF6rRJOLjCmmwTNg= -github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= -github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/redact v1.0.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/doug-martin/goqu/v9 v9.18.0 h1:/6bcuEtAe6nsSMVK/M+fOiXUNfyFF3yYtE07DBPFMYY= github.com/doug-martin/goqu/v9 v9.18.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= -github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg= -github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= -github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= -github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 h1:b5Jqi7ir58EzfeZDyp7OSYQG/IVgyY4JWfHuJUF2AZI= -github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= -github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= -github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= -github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lni/dragonboat/v3 v3.3.5 h1:5CExxfO+Kwup74Hap18M0awtGezql/Vl+Y3zegDY52I= -github.com/lni/dragonboat/v3 v3.3.5/go.mod h1:5FDHL74ORs7kI3lDDlQl6q5eBButqQpw94/QLqBjLIk= -github.com/lni/goutils v1.3.0 h1:oBhV7Z5DjNWbcy/c3fFj6qo4SnHcpyTY28qfKvLy6UM= -github.com/lni/goutils v1.3.0/go.mod h1:PUPtBAnZlRPUKWUXCsYkIRWubJbtNHpTAee0sczhlf4= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= +github.com/nats-io/nats-server/v2 v2.9.1 h1:JaP6NpCVmSu0AXgbnOkGtJovOxuf8mjNjlX3H+tSpyI= +github.com/nats-io/nats-server/v2 v2.9.1/go.mod h1:T5AEyzrnDGaseK/Y0G6e2IA5tLrHyjLOeGUALq+A8XE= +github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0 h1:dPUKD6Iv8M1y9MU8PK6H4a4/12yx5/CbaYWz/Z1arY8= +github.com/nats-io/nats.go v1.16.1-0.20220906180156-a1017eec10b0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ= github.com/samber/lo v1.27.0/go.mod h1:it33p9UtPMS7z72fP4gw/EIfQB2eI8ke7GR2wc6+Rhg= -github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= -github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/histogram v1.0.1 h1:FzA7n2Tz/wKRMejgu3PV1vw3htAklTjjuoI6z3d4KDg= -github.com/valyala/histogram v1.0.1/go.mod h1:lQy0xA4wUz2+IUnf97SivorsJIp8FxsnRd6x25q7Mto= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= +golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 h1:yuLAip3bfURHClMG9VBdzPrQvCWjWiWUTBGV+/fCbUs= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/lib/control_pane.go b/lib/control_pane.go deleted file mode 100644 index 817f2c8..0000000 --- a/lib/control_pane.go +++ /dev/null @@ -1,184 +0,0 @@ -package lib - -import ( - "os" - "strconv" - "strings" - "time" - - "github.com/gin-gonic/gin" - "github.com/rs/zerolog/log" -) - -type ControlPane struct { - engine *gin.Engine - raft *RaftServer -} - -func NewControlPane(raft *RaftServer) *ControlPane { - gin.SetMode(gin.ReleaseMode) - r := &ControlPane{ - engine: gin.New(), - raft: raft, - } - - r.engine.Use(gin.Recovery()) - r.engine.MaxMultipartMemory = 8 << 20 - routes := r.engine.Group("/", authRequired) - routes.GET("/start/cluster/:cluster/mode/:mode", r.startCluster) - routes.GET("/move/cluster/:cluster/node/:node", r.moveCluster) - routes.GET("/add/cluster/:cluster/node/:node", r.addNode) - routes.GET("/shuffle", r.shuffleNodes) - routes.GET("/info", r.clusterInfo) - routes.GET("/restore", r.restoreSnapshot) - routes.POST("/restore", r.restoreSnapshotFrom) - - return r -} - -func (c *ControlPane) Run(addr string) error { - return c.engine.Run(addr) -} - -func (c *ControlPane) startCluster(g *gin.Context) { - clusterId, err := strconv.ParseUint(g.Param("cluster"), 10, 64) - if err != nil { - log.Err(err).Msg("bind cluster error") - g.JSON(500, err.Error()) - return - } - - members := g.Query("members") - join := strings.ToLower(g.Param("mode")) == "join" - - err = c.raft.BindCluster(members, join, clusterId) - if err != nil { - log.Err(err).Msg("bind cluster error") - g.JSON(500, err.Error()) - return - } - - g.JSON(200, true) -} - -func (c *ControlPane) addNode(g *gin.Context) { - nodeId, err := strconv.ParseUint(g.Param("node"), 10, 64) - if err != nil { - log.Err(err).Msg("add node error") - g.JSON(500, err.Error()) - return - } - - clusterId, err := strconv.ParseUint(g.Param("cluster"), 10, 64) - if err != nil { - log.Err(err).Msg("add node error") - g.JSON(500, err.Error()) - return - } - - nodeAddrs := g.Query("address") - - err = c.raft.AddNode(nodeId, nodeAddrs, clusterId) - if err != nil { - log.Err(err).Msg("add node error") - g.JSON(500, err.Error()) - return - } - - g.JSON(200, true) -} - -func (c *ControlPane) moveCluster(g *gin.Context) { - clusterId, err := strconv.ParseUint(g.Param("cluster"), 10, 64) - if err != nil { - log.Err(err).Msg("move cluster error") - g.JSON(500, err.Error()) - return - } - - nodeId, err := strconv.ParseUint(g.Param("node"), 10, 64) - if err != nil { - log.Err(err).Msg("move cluster error") - g.JSON(500, err.Error()) - return - } - - err = c.raft.TransferClusters(nodeId, clusterId) - if err != nil { - log.Err(err).Msg("move cluster error") - g.JSON(500, err.Error()) - return - } - - g.JSON(200, true) -} - -func (c *ControlPane) clusterInfo(g *gin.Context) { - cmap := c.raft.GetNodeMap() - g.JSON(200, cmap) -} - -func (c *ControlPane) restoreSnapshotFrom(g *gin.Context) { - file, err := g.FormFile("dump") - if err != nil { - log.Error().Err(err).Msg("Unable to load dump file") - g.AbortWithError(400, err) - return - } - - tmpFile, err := os.CreateTemp("", "*.sqlite") - if err != nil { - g.AbortWithError(500, err) - return - } - tmpPath := tmpFile.Name() - tmpFile.Close() - - err = g.SaveUploadedFile(file, tmpPath) - if err != nil { - g.AbortWithError(500, err) - return - } - - err = c.raft.GetSnapshotStateMachine().DB.RestoreFrom(tmpPath) - if err != nil { - g.AbortWithError(500, err) - return - } - - g.JSON(200, "OK") -} - -func (c *ControlPane) restoreSnapshot(g *gin.Context) { - index, _, err := c.raft.RequestSnapshot(60 * time.Second) - if err != nil { - log.Error().Err(err).Msg("Unable to request snapshot") - _ = g.AbortWithError(500, err) - return - } - - g.JSON(200, index) -} - -func (c *ControlPane) shuffleNodes(g *gin.Context) { - err := c.raft.ShuffleCluster(c.raft.nodeID) - if err != nil { - log.Err(err).Msg("shuffle nodes error") - g.AbortWithStatus(500) - return - } - - // Time to propagate - time.Sleep(1 * time.Second) - g.JSON(200, c.raft.GetNodeMap()) -} - -func authRequired(c *gin.Context) { - header := c.Request.Header.Get("Authorization") - payload := os.Getenv("AUTH_KEY") - if header == "" || header != payload { - c.Header("WWW-Authenticate", "Basic") - c.AbortWithStatus(401) - return - } -} diff --git a/lib/log_db_script.sql b/lib/log_db_script.sql deleted file mode 100644 index 2edb4f5..0000000 --- a/lib/log_db_script.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE IF NOT EXISTS raft_info( - entry_index UNSIGNED BIG INT, - node_id UNSIGNED BIG INT NOT NULL, - cluster_id UNSIGNED BIG INT NOT NULL, - entry_type INTEGER NOT NULL, - payload BLOB -); - -CREATE INDEX IF NOT EXISTS raft_info_tpl_index - ON raft_info(node_id, cluster_id, entry_type); - -CREATE UNIQUE INDEX IF NOT EXISTS raft_info_entry_index - ON raft_info(entry_index, node_id, cluster_id, entry_type) - WHERE entry_index IS NOT NULL; \ No newline at end of file diff --git a/lib/raft.go b/lib/raft.go deleted file mode 100644 index 87a79ee..0000000 --- a/lib/raft.go +++ /dev/null @@ -1,367 +0,0 @@ -package lib - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strconv" - "strings" - "sync" - "time" - - "github.com/maxpert/marmot/db" - - "github.com/lni/dragonboat/v3" - "github.com/lni/dragonboat/v3/config" - "github.com/lni/dragonboat/v3/logger" - "github.com/lni/dragonboat/v3/raftio" - "github.com/lni/dragonboat/v3/statemachine" - "github.com/rs/zerolog/log" -) - -var MaxLogEntries = uint64(10_000) - -type RaftServer struct { - bindAddress string - nodeID uint64 - metaPath string - lock *sync.RWMutex - nodeHost *dragonboat.NodeHost - database *db.SqliteStreamDB - - clusterStateMachine map[uint64]*SQLiteStateMachine - nodeUser map[uint64]dragonboat.INodeUser - nodeMap map[uint64]map[uint64]uint64 - clusterMap map[uint64]uint64 -} - -func NewRaftServer( - bindAddress string, - nodeID uint64, - metaPath string, - database *db.SqliteStreamDB, -) *RaftServer { - logger.GetLogger("raft").SetLevel(logger.ERROR) - logger.GetLogger("rsm").SetLevel(logger.WARNING) - logger.GetLogger("transport").SetLevel(logger.ERROR) - logger.GetLogger("grpc").SetLevel(logger.WARNING) - logger.GetLogger("dragonboat").SetLevel(logger.WARNING) - logger.GetLogger("logdb").SetLevel(logger.WARNING) - logger.GetLogger("config").SetLevel(logger.WARNING) - - return &RaftServer{ - bindAddress: bindAddress, - nodeID: nodeID, - database: database, - metaPath: metaPath, - lock: &sync.RWMutex{}, - - nodeUser: map[uint64]dragonboat.INodeUser{}, - clusterMap: make(map[uint64]uint64), - nodeMap: map[uint64]map[uint64]uint64{}, - clusterStateMachine: map[uint64]*SQLiteStateMachine{}, - } -} - -func (r *RaftServer) Init() error { - r.lock.Lock() - defer r.lock.Unlock() - - metaAbsPath := fmt.Sprintf("%s/node-%d", r.metaPath, r.nodeID) - hostConfig := config.NodeHostConfig{ - WALDir: metaAbsPath, - NodeHostDir: metaAbsPath, - RTTMillisecond: 300, - RaftAddress: r.bindAddress, - RaftEventListener: r, - } - - factory := NewSQLiteLogDBFactory(r.metaPath, r.nodeID) - hostConfig.Expert = config.ExpertConfig{ - LogDBFactory: factory, - } - - nodeHost, err := dragonboat.NewNodeHost(hostConfig) - if err != nil { - return err - } - - r.nodeHost = nodeHost - return nil -} - -func (r *RaftServer) BindCluster(initMembers string, join bool, clusterIDs ...uint64) error { - initialMembers := parsePeersMap(initMembers) - if !join { - initialMembers[r.nodeID] = r.bindAddress - } - - for _, clusterID := range clusterIDs { - cfg := r.config(clusterID) - log.Debug(). - Uint64("cluster", clusterID). - Uint64("node", r.nodeID). - Msg("Starting cluster...") - err := r.nodeHost.StartOnDiskCluster(initialMembers, join, r.stateMachineFactory, cfg) - if err != nil { - return err - } - } - - return nil -} - -func (r *RaftServer) RequestSnapshot(timeout time.Duration) (uint64, uint64, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - for clusterID, nodeID := range r.GetClusterMap() { - if nodeID == r.nodeID || nodeID == 0 { - continue - } - - ret, err := r.nodeHost.SyncRequestSnapshot(ctx, clusterID, dragonboat.SnapshotOption{}) - - if err != nil { - return 0, 0, err - } - - return ret, clusterID, nil - } - - return 0, 0, nil -} - -func (r *RaftServer) AddNode(peerID uint64, address string, clusterIDs ...uint64) error { - r.lock.Lock() - defer r.lock.Unlock() - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - for _, clusterID := range clusterIDs { - mem, err := r.nodeHost.SyncGetClusterMembership(ctx, clusterID) - if err != nil { - return err - } - - err = r.nodeHost.SyncRequestAddNode(ctx, clusterID, peerID, address, mem.ConfigChangeID) - if err != nil { - return err - } - } - - return nil -} - -func (r *RaftServer) ShuffleCluster(nodes ...uint64) error { - nodeMap := r.GetNodeMap() - allNodeList := make([]uint64, 0) - if len(nodes) > 0 { - allNodeList = append(allNodeList, nodes...) - } - - for nodeID := range nodeMap { - allNodeList = append(allNodeList, nodeID) - } - - rand.Seed(time.Now().UnixNano()) - - for nodeIndex, nodeID := range allNodeList { - clusterIDs := nodeMap[nodeID] - for _, clusterID := range clusterIDs { - newOwnerNodeIndex := rand.Uint64() % uint64(len(allNodeList)) - if newOwnerNodeIndex != uint64(nodeIndex) { - newOwnerNodeID := allNodeList[newOwnerNodeIndex] - - log.Debug().Msg(fmt.Sprintf("Moving %v from %v to %v", clusterID, nodeID, newOwnerNodeID)) - if err := r.TransferClusters(newOwnerNodeID, clusterID); err != nil { - return err - } - } - } - } - - return nil -} - -func (r *RaftServer) GetSnapshotStateMachine() *SQLiteStateMachine { - r.lock.RLock() - defer r.lock.RUnlock() - - for _, sm := range r.clusterStateMachine { - if sm.IsSnapshotEnabled() { - return sm - } - } - - return nil -} - -func (r *RaftServer) GetNodeMap() map[uint64][]uint64 { - nodeMap := map[uint64][]uint64{} - - for nodeID, clusterMap := range r.nodeMap { - clusters := make([]uint64, 0) - for clusterID := range clusterMap { - clusters = append(clusters, clusterID) - } - - nodeMap[nodeID] = clusters - } - - return nodeMap -} - -func (r *RaftServer) TransferClusters(toPeerID uint64, clusterIDs ...uint64) error { - for _, cluster := range clusterIDs { - err := r.nodeHost.RequestLeaderTransfer(cluster, toPeerID) - if err != nil { - return err - } - } - - return nil -} - -func (r *RaftServer) GetActiveClusters() []uint64 { - r.lock.RLock() - defer r.lock.RUnlock() - - ret := make([]uint64, 0) - for clusterID, nodeID := range r.clusterMap { - if nodeID != 0 { - ret = append(ret, clusterID) - } - } - - return ret -} - -func (r *RaftServer) GetClusterMap() map[uint64]uint64 { - r.lock.RLock() - defer r.lock.RUnlock() - return r.clusterMap -} - -func (r *RaftServer) Propose(clusterKey uint64, data []byte, dur time.Duration) (*dragonboat.RequestResult, error) { - clusterIds := r.GetActiveClusters() - if len(clusterIds) == 0 { - return nil, errors.New("cluster not ready") - } - - clusterIndex := clusterKey % uint64(len(clusterIds)) - clusterId := clusterIds[clusterIndex] - nodeUser, err := r.getNodeUser(clusterId) - if err != nil { - return nil, err - } - - session := r.nodeHost.GetNoOPSession(clusterId) - req, err := nodeUser.Propose(session, data, dur) - if err != nil { - return nil, err - } - - res := <-req.ResultC() - return &res, err -} - -func (r *RaftServer) LeaderUpdated(info raftio.LeaderInfo) { - r.lock.Lock() - defer r.lock.Unlock() - - if info.LeaderID == 0 { - previousLeader := r.clusterMap[info.ClusterID] - delete(r.clusterMap, info.ClusterID) - r.mutateNodeMap(previousLeader, func(m map[uint64]uint64) { - delete(m, info.ClusterID) - }) - } else { - r.clusterMap[info.ClusterID] = info.LeaderID - r.mutateNodeMap(info.LeaderID, func(m map[uint64]uint64) { - m[info.ClusterID] = info.Term - }) - } -} - -func (r *RaftServer) mutateNodeMap(nodeID uint64, f func(map[uint64]uint64)) { - m, ok := r.nodeMap[nodeID] - if !ok { - m = make(map[uint64]uint64, 0) - } - - f(m) - r.nodeMap[nodeID] = m - - if len(m) <= 0 { - delete(r.nodeMap, nodeID) - } -} - -func (r *RaftServer) stateMachineFactory(clusterID uint64, nodeID uint64) statemachine.IOnDiskStateMachine { - r.lock.Lock() - defer r.lock.Unlock() - - sm, ok := r.clusterStateMachine[clusterID] - if !ok { - sm = NewDBStateMachine(clusterID, nodeID, r.database, r.metaPath, clusterID == 1) - r.clusterStateMachine[clusterID] = sm - } - - return sm -} - -func (r *RaftServer) getNodeUser(clusterID uint64) (dragonboat.INodeUser, error) { - r.lock.RLock() - if val, ok := r.nodeUser[clusterID]; ok { - r.lock.RUnlock() - return val, nil - } - - r.lock.RUnlock() - r.lock.Lock() - defer r.lock.Unlock() - - val, err := r.nodeHost.GetNodeUser(clusterID) - if err != nil { - return nil, err - } - r.nodeUser[clusterID] = val - return val, nil -} - -func (r *RaftServer) config(clusterID uint64) config.Config { - return config.Config{ - NodeID: r.nodeID, - ClusterID: clusterID, - ElectionRTT: 10, - HeartbeatRTT: 1, - CheckQuorum: true, - SnapshotEntries: MaxLogEntries, - CompactionOverhead: 0, - EntryCompressionType: config.Snappy, - SnapshotCompressionType: config.Snappy, - } -} - -func parsePeersMap(peersAddrs string) map[uint64]string { - peersMap := make(map[uint64]string) - if peersAddrs == "" { - return peersMap - } - - for _, peer := range strings.Split(peersAddrs, ",") { - peerInf := strings.Split(peer, "@") - peerShard, err := strconv.ParseUint(peerInf[0], 10, 64) - if err != nil { - continue - } - - peersMap[peerShard] = peerInf[1] - } - - log.Debug().Msg(fmt.Sprintf("Peer map %v", peersMap)) - return peersMap -} diff --git a/lib/replication_event.go b/lib/replication_event.go new file mode 100644 index 0000000..dbef5a8 --- /dev/null +++ b/lib/replication_event.go @@ -0,0 +1,16 @@ +package lib + +import "github.com/fxamacker/cbor/v2" + +type ReplicationEvent[T any] struct { + FromNodeId uint64 + Payload *T +} + +func (e *ReplicationEvent[T]) Marshal() ([]byte, error) { + return cbor.Marshal(e) +} + +func (e *ReplicationEvent[T]) Unmarshal(data []byte) error { + return cbor.Unmarshal(data, e) +} diff --git a/lib/replicator.go b/lib/replicator.go new file mode 100644 index 0000000..ed9b27d --- /dev/null +++ b/lib/replicator.go @@ -0,0 +1,107 @@ +package lib + +import ( + "fmt" + "time" + + "github.com/nats-io/nats.go" + "github.com/rs/zerolog/log" +) + +const DefaultUrl = nats.DefaultURL +const MaxLogEntries = int64(1024) +const clusterPrefix = "marmot.cluster." + +type Replicator struct { + nodeID uint64 + shards uint64 + + client *nats.Conn + js nats.JetStream +} + +func NewReplicator(nodeID uint64, natsServer string, shards uint64) (*Replicator, error) { + nc, err := nats.Connect(natsServer, nats.Name(fmt.Sprintf("node-%d", nodeID))) + if err != nil { + return nil, err + } + + js, err := nc.JetStream() + if err != nil { + return nil, err + } + + streamCfg := &nats.StreamConfig{ + Name: "marmot", + Subjects: []string{clusterPrefix + "*"}, + Discard: nats.DiscardOld, + MaxMsgs: MaxLogEntries, + Storage: nats.FileStorage, + Retention: nats.InterestPolicy, + AllowDirect: true, + MaxConsumers: -1, + Duplicates: 0, + DenyDelete: true, + Replicas: 2, + } + + info, err := js.StreamInfo("marmot") + if err != nil { + info, err = js.AddStream(streamCfg) + } else { + info, err = js.UpdateStream(streamCfg) + } + + log.Info().Msg(fmt.Sprintf("Stream info %v", info)) + return &Replicator{ + client: nc, + nodeID: nodeID, + js: js, + shards: shards, + }, nil +} + +func (r *Replicator) Publish(hash uint64, payload []byte) error { + shard := hash % r.shards + ack, err := r.js.Publish(fmt.Sprintf("%s%d", clusterPrefix, shard), payload) + if err != nil { + return err + } + + log.Debug().Uint64("seq", ack.Sequence).Msg(ack.Stream) + return nil +} + +func (r *Replicator) Listen(callback func(payload []byte) error) error { + sub, err := r.js.SubscribeSync(clusterPrefix + "*") + if err != nil { + return err + } + defer sub.Unsubscribe() + + for sub.IsValid() { + msg, err := sub.NextMsg(1 * time.Second) + + if err == nats.ErrTimeout { + continue + } + + if err != nil { + return err + } + + log.Debug().Str("sub", msg.Subject).Msg("Msg...") + + err = callback(msg.Data) + if err != nil { + return err + } + + err = msg.Ack() + if err != nil { + return err + } + } + + return nil +} diff --git a/lib/sqlite_log_db.go b/lib/sqlite_log_db.go deleted file mode 100644 index d8a7716..0000000 --- a/lib/sqlite_log_db.go +++ /dev/null @@ -1,676 +0,0 @@ -package lib - -import ( - "database/sql" - "errors" - "fmt" - "math" - "time" - - _ "embed" - - "github.com/doug-martin/goqu/v9" - "github.com/lni/dragonboat/v3/config" - "github.com/lni/dragonboat/v3/raftio" - "github.com/lni/dragonboat/v3/raftpb" - _ "github.com/mattn/go-sqlite3" - "github.com/maxpert/marmot/db" - "github.com/rs/zerolog/log" - "github.com/samber/lo" -) - -type raftInfoEntryType = int16 - -type SQLiteLogDB struct { - name string - db *goqu.Database -} - -type raftInfoEntry struct { - NodeId uint64 `db:"node_id"` - ClusterId uint64 `db:"cluster_id"` - Index *uint64 `db:"entry_index"` - Type raftInfoEntryType `db:"entry_type"` - Payload []byte `db:"payload"` -} - -type SQLiteLogDBFactory struct { - metaPath string - nodeID uint64 - path string -} - -const raftInfoTable = "raft_info" - -const ( - State raftInfoEntryType = 1 - Entry raftInfoEntryType = 2 - Bootstrap raftInfoEntryType = 3 - Snapshot raftInfoEntryType = 4 -) - -//go:embed log_db_script.sql -var logDBScript string -var errIndexNotApplicable = errors.New("index can not be applied for this save type") - -func NewSQLiteLogDBFactory(metaPath string, nodeID uint64) *SQLiteLogDBFactory { - path := fmt.Sprintf("%s/logdb-%d.sqlite?_journal=wal", metaPath, nodeID) - return &SQLiteLogDBFactory{ - metaPath: metaPath, - nodeID: nodeID, - path: path, - } -} - -func (f *SQLiteLogDBFactory) Name() string { - return f.metaPath -} - -func (f *SQLiteLogDBFactory) Create( - _ config.NodeHostConfig, - _ config.LogDBCallback, - _ []string, - _ []string, -) (raftio.ILogDB, error) { - logDB, err := newSQLiteLogDB(f.path) - if err != nil { - return nil, err - } - - return logDB, err -} - -func newSQLiteLogDB(path string) (*SQLiteLogDB, error) { - conn, err := sql.Open("sqlite3", path) - if err != nil { - return nil, err - } - - _, err = conn.Exec(logDBScript) - if err != nil { - return nil, err - } - - return &SQLiteLogDB{ - name: path, - db: goqu.New("sqlite", conn), - }, nil -} - -func (s *SQLiteLogDB) Name() string { - return s.name -} - -func (s *SQLiteLogDB) Close() { -} - -func (s *SQLiteLogDB) BinaryFormat() uint32 { - return 1 -} - -// ListNodeInfo lists all available NodeInfo found in the log DB. -func (s *SQLiteLogDB) ListNodeInfo() ([]raftio.NodeInfo, error) { - arr := make([]raftInfoEntry, 0) - err := s.db.Select("node_id", "column_id"). - Distinct(). - From(raftInfoTable). - Prepared(true). - ScanStructs(&arr) - if err != nil { - return nil, err - } - - nodes := lo.Map(arr, func(e raftInfoEntry, i int) raftio.NodeInfo { - return raftio.NodeInfo{NodeID: e.NodeId, ClusterID: e.ClusterId} - }) - - return nodes, nil -} - -func (s *SQLiteLogDB) SaveBootstrapInfo(clusterID uint64, nodeID uint64, bootstrap raftpb.Bootstrap) error { - data, err := bootstrap.Marshal() - if err != nil { - return err - } - - _, err = s.db.Insert(raftInfoTable). - Rows(&raftInfoEntry{ - NodeId: nodeID, - ClusterId: clusterID, - Type: Bootstrap, - Payload: data, - }). - Prepared(true). - Executor(). - Exec() - - return err -} - -func (s *SQLiteLogDB) GetBootstrapInfo(clusterID uint64, nodeID uint64) (raftpb.Bootstrap, error) { - data := raftInfoEntry{} - bs := raftpb.Bootstrap{} - hasRow, err := s.db. - From(raftInfoTable). - Where(goqu.Ex{"cluster_id": clusterID, "node_id": nodeID, "entry_type": Bootstrap}). - Prepared(true). - ScanStruct(&data) - - if err != nil { - return bs, err - } - - if !hasRow { - return bs, raftio.ErrNoBootstrapInfo - } - - err = bs.Unmarshal(data.Payload) - if err != nil { - return bs, err - } - - return bs, nil -} - -func (s *SQLiteLogDB) SaveRaftState(updates []raftpb.Update, _ uint64) error { - return s.db.WithTx(func(tx *goqu.TxDatabase) error { - for _, upd := range updates { - if !raftpb.IsEmptyState(upd.State) { - err := saveInfoTuple(tx, nil, upd.NodeID, upd.ClusterID, State, upd.State.Marshal) - if err != nil { - return err - } - } - - if !raftpb.IsEmptySnapshot(upd.Snapshot) { - if len(upd.EntriesToSave) > 0 { - lastIndex := upd.EntriesToSave[len(upd.EntriesToSave)-1].Index - if upd.Snapshot.Index > lastIndex { - return fmt.Errorf("max index not handled, %d, %d", upd.Snapshot.Index, lastIndex) - } - } - - err := saveInfoTuple(tx, &upd.Snapshot.Index, upd.NodeID, upd.ClusterID, Snapshot, upd.Snapshot.Marshal) - if err != nil { - return err - } - } - - for _, entry := range upd.EntriesToSave { - err := saveInfoTuple(tx, &entry.Index, upd.NodeID, upd.ClusterID, Entry, entry.Marshal) - if err != nil { - return err - } - } - } - - return nil - }) -} - -func (s *SQLiteLogDB) RemoveNodeData(clusterID uint64, nodeID uint64) error { - _, err := s.db.Delete(raftInfoTable).Where(goqu.Ex{ - "cluster_id": clusterID, - "node_id": nodeID, - }).Prepared(true).Executor().Exec() - - if err != nil { - return err - } - - return err -} - -// IterateEntries returns the continuous Raft log entries of the specified -// Raft node between the index value range of [low, high) up to a max size -// limit of maxSize bytes. It returns the located log entries, their total -// size in bytes and the occurred error. -func (s *SQLiteLogDB) IterateEntries( - entries []raftpb.Entry, - size uint64, - clusterID uint64, - nodeID uint64, - low uint64, - high uint64, - maxSize uint64, -) ([]raftpb.Entry, uint64, error) { - logger := log.With(). - Uint64("low", low). - Uint64("size", size). - Uint64("high", high). - Uint64("node_id", nodeID). - Uint64("max_size", maxSize). - Uint64("cluster_id", clusterID). - Uint64("entries", uint64(len(entries))). - Logger() - - min, count, err := s.getEntryRange(nodeID, clusterID, nil) - if err == raftio.ErrNoSavedLog { - logger.Warn().Msg("No entries...") - return entries, size, nil - } - - if err != nil { - logger.Warn().Err(err).Msg("Range error") - return entries, size, err - } - - logger = logger.With().Uint64("min", min).Uint64("max", min+count-1).Logger() - rows, err := s.db. - Select("payload"). - From(raftInfoTable). - Where( - goqu.C("node_id").Eq(nodeID), - goqu.C("cluster_id").Eq(clusterID), - goqu.C("entry_type").Eq(Entry), - goqu.C("entry_index").Gte(low), - goqu.C("entry_index").Lt(high)). - Order(goqu.C("entry_index").Asc()). - Prepared(true). - Executor().Query() - - if err != nil { - return entries, size, err - } - eRow := &db.EnhancedRows{Rows: rows} - defer eRow.Finalize() - - expectedIndex := low - buff := make([]byte, 0) - for eRow.Next() { - err = eRow.Scan(&buff) - if err != nil { - return entries, size, err - } - - e := raftpb.Entry{} - err = e.Unmarshal(buff) - if err != nil { - return entries, size, err - } - if e.Index != expectedIndex { - logger.Warn().Msg(fmt.Sprintf("Index mismatch %d != %d", e.Index, expectedIndex)) - break - } - - size += uint64(e.SizeUpperLimit()) - entries = append(entries, e) - expectedIndex++ - - if size >= maxSize { - logger.Trace().Msg(fmt.Sprintf("Size mismatch %d != %d", size, maxSize)) - break - } - } - - logger.Trace().Msg("Scan complete") - return entries, size, nil -} - -func (s *SQLiteLogDB) ReadRaftState(clusterID uint64, nodeID uint64, snapshotIndex uint64) (raftio.RaftState, error) { - entry := raftInfoEntry{} - ret := raftio.RaftState{} - log.Trace().Msg(fmt.Sprintf("ReadRaftState %d %d %d", clusterID, nodeID, snapshotIndex)) - - minIndex, entriesCount, err := s.getEntryRange(nodeID, clusterID, &snapshotIndex) - if err != nil { - return ret, err - } - - ok, err := s.db.From(raftInfoTable). - Where(goqu.Ex{ - "node_id": nodeID, - "cluster_id": clusterID, - "entry_type": State, - }).ScanStruct(&entry) - - if err != nil { - return ret, err - } - - if !ok { - return ret, raftio.ErrNoSavedLog - } - - err = ret.State.Unmarshal(entry.Payload) - if err != nil { - return ret, err - } - - ret.FirstIndex = minIndex - ret.EntryCount = entriesCount - - return ret, nil -} - -// RemoveEntriesTo removes entries associated with the specified Raft node up -// to the specified index. -func (s *SQLiteLogDB) RemoveEntriesTo(clusterID uint64, nodeID uint64, index uint64) error { - return s.db.WithTx(func(tx *goqu.TxDatabase) error { - log.Trace().Msg(fmt.Sprintf("RemoveEntriesTo c: %d n: %d i: %d", clusterID, nodeID, index)) - return deleteInfoTuple(tx, nodeID, clusterID, Entry, []goqu.Expression{ - goqu.C("entry_index").Lt(index), - }) - }) -} - -func (s *SQLiteLogDB) CompactEntriesTo(clusterID uint64, nodeID uint64, index uint64) (<-chan struct{}, error) { - ch := make(chan struct{}) - log.Trace().Msg(fmt.Sprintf("CompactEntriesTo c: %d n: %d i: %d", clusterID, nodeID, index)) - - defer func() { - var rows *sql.Rows - var err error - - for { - if rows != nil { - _ = rows.Close() - } - - rows, err = s.db.Query("PRAGMA wal_checkpoint(TRUNCATE)") - if err != nil { - log.Error().Err(err).Msg("Unable to compact checkpoint") - break - } - - rows.Next() - - var busy, logi, checkpointed int64 - err = rows.Scan(&busy, &logi, &checkpointed) - if err != nil { - log.Error().Err(err).Msg("Unable to read checkpoint data") - break - } - - if busy == 0 { - log.Info(). - Int64("log_index", logi). - Int64("checkpointed", checkpointed). - Msg("Checkpoint complete") - break - } - - time.Sleep(200 * time.Millisecond) - } - - if rows != nil { - _ = rows.Close() - } - close(ch) - }() - - return ch, nil -} - -func (s *SQLiteLogDB) SaveSnapshots(updates []raftpb.Update) error { - return s.db.WithTx(func(tx *goqu.TxDatabase) error { - for _, upd := range updates { - err := saveInfoTuple(tx, &upd.Snapshot.Index, upd.NodeID, upd.ClusterID, Snapshot, upd.Snapshot.Marshal) - if err != nil { - return err - } - } - - return nil - }) -} - -func (s *SQLiteLogDB) DeleteSnapshot(clusterID uint64, nodeID uint64, index uint64) error { - return s.db.WithTx(func(tx *goqu.TxDatabase) error { - return deleteInfoTuple(tx, nodeID, clusterID, Snapshot, []goqu.Expression{ - goqu.C("entry_index").Eq(index), - }) - }) -} - -func (s *SQLiteLogDB) ListSnapshots(clusterID uint64, nodeID uint64, index uint64) ([]raftpb.Snapshot, error) { - exps := []goqu.Expression{ - goqu.C("node_id").Eq(nodeID), - goqu.C("cluster_id").Eq(clusterID), - goqu.C("entry_type").Eq(Snapshot), - } - - if index != math.MaxUint64 { - exps = append(exps, goqu.C("entry_index").Lte(index)) - } - - rows, err := s.db. - Select("payload"). - From(raftInfoTable). - Where(exps...). - Order(goqu.C("entry_index").Asc()). - Prepared(true). - Executor(). - Query() - - if err != nil { - return nil, err - } - - eRows := db.EnhancedRows{Rows: rows} - defer eRows.Finalize() - - ret := make([]raftpb.Snapshot, 0) - buf := make([]byte, 0) - for eRows.Next() { - err = eRows.Scan(&buf) - if err != nil { - return nil, err - } - - snp := raftpb.Snapshot{} - err = snp.Unmarshal(buf) - if err != nil { - return nil, err - } - - ret = append(ret, snp) - } - - return ret, nil -} - -func (s *SQLiteLogDB) ImportSnapshot(snp raftpb.Snapshot, nodeID uint64) error { - return s.db.WithTx(func(tx *goqu.TxDatabase) error { - if raftpb.IsEmptySnapshot(snp) { - return nil - } - - // Replace Bootstrap - err := deleteInfoTuple(tx, nodeID, snp.ClusterId, Bootstrap, []goqu.Expression{}) - if err != nil { - return err - } - - bootstrap := raftpb.Bootstrap{ - Join: true, - Type: snp.Type, - } - err = saveInfoTuple(tx, nil, nodeID, snp.ClusterId, Bootstrap, bootstrap.Marshal) - if err != nil { - return err - } - - // Replace state - err = deleteInfoTuple(tx, nodeID, snp.ClusterId, State, []goqu.Expression{}) - if err != nil { - return err - } - - state := raftpb.State{ - Term: snp.Term, - Commit: snp.Index, - } - err = saveInfoTuple(tx, nil, nodeID, snp.ClusterId, State, state.Marshal) - if err != nil { - return err - } - - // Delete snapshot log entries ahead of index - err = deleteInfoTuple(tx, nodeID, snp.ClusterId, Snapshot, []goqu.Expression{ - goqu.C("entry_index").Gte(snp.Index), - }) - if err != nil { - return err - } - - err = saveInfoTuple(tx, &snp.Index, nodeID, snp.ClusterId, Snapshot, snp.Marshal) - if err != nil { - return err - } - - return nil - }) -} - -func (s *SQLiteLogDB) getEntryRange(nodeID, clusterID uint64, index *uint64) (uint64, uint64, error) { - count := uint64(0) - ok, err := s.db.From(raftInfoTable). - Select(goqu.COUNT("entry_index")). - Where(goqu.Ex{"node_id": nodeID, "cluster_id": clusterID, "entry_type": Entry}). - Prepared(true). - Executor(). - ScanVal(&count) - - if err != nil { - return 0, 0, err - } - - if !ok || count == 0 { - return 0, 0, raftio.ErrNoSavedLog - } - - if index == nil { - return 0, count, nil - } - - max := uint64(0) - _, err = s.db.From(raftInfoTable). - Select(goqu.MAX("entry_index")). - Where(goqu.Ex{"node_id": nodeID, "cluster_id": clusterID, "entry_type": Entry}). - Prepared(true). - Executor(). - ScanVal(&max) - - if err != nil { - return 0, 0, err - } - - if max == *index { - return max, 0, nil - } - - min := uint64(0) - _, err = s.db.From(raftInfoTable). - Select(goqu.MIN("entry_index")). - Where( - goqu.Ex{"node_id": nodeID, "cluster_id": clusterID, "entry_type": Entry}, - goqu.C("entry_index").Gte(*index), - goqu.C("entry_index").Lte(max), - ). - Prepared(true). - Executor(). - ScanVal(&min) - - if err != nil { - return 0, 0, err - } - - return min, max - min + 1, nil -} - -func deleteInfoTuple( - db *goqu.TxDatabase, - nodeID uint64, - clusterID uint64, - entryType raftInfoEntryType, - additionalExpressions []goqu.Expression, -) error { - - logger := log.With(). - Uint64("node_id", nodeID). - Uint64("cluster_id", clusterID). - Uint64("entry_type", uint64(entryType)). - Logger() - - for i, x := range additionalExpressions { - logger = logger.With().Str(fmt.Sprintf("exp[%d]", i), fmt.Sprintf("%v", x)).Logger() - } - - exps := []goqu.Expression{ - goqu.C("node_id").Eq(nodeID), - goqu.C("cluster_id").Eq(clusterID), - goqu.C("entry_type").Eq(entryType), - } - - exps = append(exps, additionalExpressions...) - - logger.Trace().Msg("deleteInfoTuple") - _, err := db.Delete(raftInfoTable). - Where(exps...). - Prepared(true). - Executor(). - Exec() - - if err != nil { - return err - } - return nil -} - -func saveInfoTuple( - tx *goqu.TxDatabase, - index *uint64, - nodeID uint64, - clusterID uint64, - entryType raftInfoEntryType, - f func() ([]byte, error), -) error { - // Assert that BootStrap and State has no index - if entryType == Bootstrap || entryType == State { - if index != nil { - return errIndexNotApplicable - } - } - - data, err := f() - if err != nil { - return err - } - - exps := []goqu.Expression{ - goqu.C("node_id").Eq(nodeID), - goqu.C("cluster_id").Eq(clusterID), - goqu.C("entry_type").Eq(entryType), - } - - logger := log.With(). - Uint64("node_id", nodeID). - Uint64("cluster_id", clusterID). - Uint64("entry_type", uint64(entryType)). - Logger() - - if index != nil { - exps = append(exps, goqu.C("entry_index").Eq(*index)) - logger = logger.With().Uint64("index", *index).Logger() - } else { - exps = append(exps, goqu.C("entry_index").IsNull()) - logger = logger.With().Int("index", -1).Logger() - } - - _, err = tx.Delete(raftInfoTable).Where(exps...).Prepared(true).Executor().Exec() - if err != nil { - return err - } - - _, err = tx.Insert(raftInfoTable).Rows(goqu.Record{ - "node_id": nodeID, - "entry_index": index, - "cluster_id": clusterID, - "entry_type": entryType, - "payload": data, - }).Prepared(true).Executor().Exec() - - logger.Trace().Err(err).Msg("saveTupleInfo") - return err -} diff --git a/lib/sqlite_state_machine.go b/lib/sqlite_state_machine.go deleted file mode 100644 index c9c22be..0000000 --- a/lib/sqlite_state_machine.go +++ /dev/null @@ -1,387 +0,0 @@ -package lib - -import ( - "fmt" - "io" - "os" - "path" - "sync" - - "github.com/fxamacker/cbor/v2" - sm "github.com/lni/dragonboat/v3/statemachine" - "github.com/maxpert/marmot/db" - "github.com/rs/zerolog/log" -) - -type snapshotState = uint8 - -type appliedIndexInfo struct { - Index uint64 -} - -type stateSaveInfo struct { - appliedIndex appliedIndexInfo - dbPath string -} - -type SQLiteStateMachine struct { - NodeID uint64 - ClusterID uint64 - DB *db.SqliteStreamDB - RaftPath string - - enableSnapshots bool - snapshotLock *sync.Mutex - snapshotState snapshotState - applied *appliedIndexInfo -} - -type ReplicationEvent[T any] struct { - FromNodeId uint64 - Payload *T -} - -const ( - snapshotNotInitialized snapshotState = 0 - snapshotSaved snapshotState = 1 - snapshotRestored snapshotState = 2 -) - -func (e *ReplicationEvent[T]) Marshal() ([]byte, error) { - return cbor.Marshal(e) -} - -func (e *ReplicationEvent[T]) Unmarshal(data []byte) error { - return cbor.Unmarshal(data, e) -} - -func NewDBStateMachine( - clusterID, nodeID uint64, - db *db.SqliteStreamDB, - path string, - enableSnapshots bool, -) *SQLiteStateMachine { - return &SQLiteStateMachine{ - DB: db, - NodeID: nodeID, - ClusterID: clusterID, - RaftPath: path, - - enableSnapshots: enableSnapshots, - snapshotLock: &sync.Mutex{}, - snapshotState: snapshotNotInitialized, - applied: &appliedIndexInfo{Index: 0}, - } -} - -func (ssm *SQLiteStateMachine) Open(_ <-chan struct{}) (uint64, error) { - err := ssm.readIndex() - if err != nil { - return 0, err - } - - return ssm.applied.Index, nil -} - -func (ssm *SQLiteStateMachine) Update(entries []sm.Entry) ([]sm.Entry, error) { - for _, entry := range entries { - event := &ReplicationEvent[db.ChangeLogEvent]{} - if err := event.Unmarshal(entry.Cmd); err != nil { - log.Error().Err(err).Msg("Unable to unmarshal command") - return nil, err - } - - logger := log.With(). - Int64("table_id", event.Payload.Id). - Str("table_name", event.Payload.TableName). - Str("type", event.Payload.Type). - Logger() - - err := ssm.DB.Replicate(event.Payload) - if err != nil { - logger.Error().Err(err).Msg("Row not replicated...") - return nil, err - } - - ssm.applied.Index = entry.Index - if err := ssm.saveIndex(); err != nil { - return nil, err - } - - entry.Result = sm.Result{Value: entry.Index} - } - - return entries, nil -} - -func (ssm *SQLiteStateMachine) Sync() error { - return nil -} - -func (ssm *SQLiteStateMachine) PrepareSnapshot() (interface{}, error) { - log.Debug(). - Uint64("cluster", ssm.ClusterID). - Uint64("node", ssm.NodeID). - Bool("enabled", ssm.enableSnapshots). - Msg("Preparing snapshot...") - - if !ssm.enableSnapshots { - return stateSaveInfo{dbPath: "", appliedIndex: *ssm.applied}, nil - } - - bkFileDir, err := ssm.getSnapshotDir() - if err != nil { - return nil, err - } - - bkFilePath := path.Join(bkFileDir, "backup.sqlite") - err = ssm.DB.BackupTo(bkFilePath) - if err != nil { - return nil, err - } - - return stateSaveInfo{dbPath: bkFilePath, appliedIndex: *ssm.applied}, nil -} - -func (ssm *SQLiteStateMachine) SaveSnapshot(st interface{}, writer io.Writer, _ <-chan struct{}) error { - log.Debug(). - Uint64("cluster", ssm.ClusterID). - Uint64("node", ssm.NodeID). - Bool("enabled", ssm.enableSnapshots). - Msg("Saving snapshot...") - - stInfo, ok := st.(stateSaveInfo) - if !ok { - return fmt.Errorf(fmt.Sprintf("invalid save state info %v", st)) - } - - mBytes, err := cbor.Marshal(stInfo.appliedIndex) - err = writeUint32(writer, uint32(len(mBytes))) - if err != nil { - return err - } - - _, err = writer.Write(mBytes) - if err != nil { - return err - } - - // Write length of filepath as indicator for following up stream - err = writeUint32(writer, uint32(len(stInfo.dbPath))) - if err != nil { - return err - } - - if stInfo.dbPath != "" { - filepath := stInfo.dbPath - fi, err := os.Open(filepath) - if err != nil { - return err - } - defer ssm.cleanup(fi, filepath) - - _, err = io.Copy(writer, fi) - if err != nil { - return err - } - } - - ssm.snapshotLock.Lock() - defer ssm.snapshotLock.Unlock() - ssm.snapshotState = snapshotSaved - return nil -} - -func (ssm *SQLiteStateMachine) RecoverFromSnapshot(reader io.Reader, _ <-chan struct{}) error { - log.Debug(). - Uint64("cluster", ssm.ClusterID). - Uint64("node", ssm.NodeID). - Bool("enabled", ssm.enableSnapshots). - Msg("Recovering from snapshot...") - - appIndex := appliedIndexInfo{} - buffLen, err := readUint32(reader) - if err != nil { - return err - } - - dec := cbor.NewDecoder(io.LimitReader(reader, int64(buffLen))) - err = dec.Decode(&appIndex) - if err != nil { - return err - } - - hasData, err := readUint32(reader) - if err != nil { - return err - } - - ssm.snapshotLock.Lock() - defer ssm.snapshotLock.Unlock() - if hasData != 0 { - basePath, err := ssm.getSnapshotDir() - if err != nil { - return err - } - - filepath := path.Join(basePath, "restore.sqlite") - fo, err := os.OpenFile(filepath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return err - } - defer ssm.cleanup(fo, filepath) - - _, err = io.Copy(fo, reader) - if err != nil { - return err - } - - // Flush file contents before handing off - err = fo.Sync() - if err != nil { - return err - } - - err = ssm.importSnapshot(filepath) - if err != nil { - return err - } - } - - ssm.applied = &appIndex - return ssm.saveIndex() -} - -func (ssm *SQLiteStateMachine) Lookup(_ interface{}) (interface{}, error) { - return 0, nil -} - -func (ssm *SQLiteStateMachine) IsSnapshotEnabled() bool { - return ssm.enableSnapshots -} - -func (ssm *SQLiteStateMachine) HasRestoredSnapshot() bool { - ssm.snapshotLock.Lock() - defer ssm.snapshotLock.Unlock() - - return ssm.snapshotState == snapshotRestored -} - -func (ssm *SQLiteStateMachine) HasSavedSnapshot() bool { - ssm.snapshotLock.Lock() - defer ssm.snapshotLock.Unlock() - - return ssm.snapshotState == snapshotSaved -} - -func (ssm *SQLiteStateMachine) Close() error { - return nil -} - -func (ssm *SQLiteStateMachine) importSnapshot(filepath string) error { - log.Info().Str("path", filepath).Msg("Importing...") - err := ssm.DB.RestoreFrom(filepath) - if err != nil { - return err - } - - log.Info().Str("path", filepath).Msg("Snapshot imported") - ssm.snapshotState = snapshotRestored - return nil -} - -func (ssm *SQLiteStateMachine) getSnapshotDir() (string, error) { - tmpPath := path.Join(ssm.RaftPath, fmt.Sprintf("marmot-%d-%d", ssm.ClusterID, ssm.NodeID)) - err := os.MkdirAll(tmpPath, 0744) - if err != nil { - log.Error().Err(err).Msg("Unable to create directory for snapshot") - return "", err - } - - return tmpPath, nil -} - -func (ssm *SQLiteStateMachine) cleanup(f *os.File, filepath string) { - if err := f.Close(); err != nil { - log.Warn().Err(err).Str("path", filepath).Msg("Unable to close snapshot file") - } - - err := os.Remove(filepath) - if err != nil { - log.Error().Err(err).Str("path", filepath).Msg("Unable to cleanup snapshot file") - } -} - -func (ssm *SQLiteStateMachine) saveIndex() error { - filepath, err := ssm.getIndexFilePath() - if err != nil { - return err - } - - fo, err := os.OpenFile(filepath, os.O_RDWR|os.O_CREATE|os.O_SYNC, 0644) - if err != nil { - return err - } - defer func() { _ = fo.Close() }() - - _, err = fo.Seek(0, io.SeekStart) - if err != nil { - return err - } - - b, err := cbor.Marshal(ssm.applied) - if err != nil { - return err - } - - _, err = fo.Write(b) - if err != nil { - return err - } - - err = fo.Sync() - if err != nil { - return err - } - - return nil -} - -func (ssm *SQLiteStateMachine) readIndex() error { - filepath, err := ssm.getIndexFilePath() - if err != nil { - return err - } - - fi, err := os.OpenFile(filepath, os.O_RDONLY, 0) - if err != nil { - if os.IsNotExist(err) { - return nil - } - - return err - } - defer func() { _ = fi.Close() }() - - b, err := io.ReadAll(fi) - if err != nil { - return err - } - - err = cbor.Unmarshal(b, ssm.applied) - if err != nil { - return err - } - - return nil -} - -func (ssm *SQLiteStateMachine) getIndexFilePath() (string, error) { - basePath, err := ssm.getSnapshotDir() - if err != nil { - return "", err - } - - filepath := path.Join(basePath, "current-index") - return filepath, nil -} diff --git a/marmot.go b/marmot.go index 646913c..47b0a79 100644 --- a/marmot.go +++ b/marmot.go @@ -1,11 +1,8 @@ package main import ( - "errors" "flag" - "fmt" "math/rand" - "time" "github.com/maxpert/marmot/db" "github.com/maxpert/marmot/lib" @@ -17,14 +14,9 @@ import ( func main() { cleanup := flag.Bool("cleanup", false, "Cleanup all trigger hooks for marmot") dbPathString := flag.String("db-path", "/tmp/marmot.db", "Path to SQLite database") - metaPath := flag.String("raft-path", "/tmp/raft", "Path to save raft information") nodeID := flag.Uint64("node-id", rand.Uint64(), "Node ID") - followFlag := flag.Bool("follow", false, "Start Raft in follower mode") - shards := flag.Uint64("shards", 16, "Total number of shards for this instance") - maxLogEntries := flag.Uint64("max-log-entries", 1024, "Total number of log entries per shard before snapshotting") - bindAddress := flag.String("bind", "0.0.0.0:8160", "Bind address for Raft server") - bindPane := flag.String("bind-pane", "localhost:6010", "Bind address for control pane server") - initialAddrs := flag.String("bootstrap", "", "@IP:PORT list of initial nodes separated by comma (,)") + natsAddr := flag.String("nats-url", lib.DefaultUrl, "NATS server URL") + shards := flag.Uint64("shards", 512, "Number of stream shards to distribute change log on") verbose := flag.Bool("verbose", false, "Log debug level") flag.Parse() @@ -36,14 +28,14 @@ func main() { log.Debug().Str("path", *dbPathString).Msg("Opening database") tableNames, err := db.GetAllDBTables(*dbPathString) - srcDb, err := db.OpenStreamDB(*dbPathString, tableNames) + streamDB, err := db.OpenStreamDB(*dbPathString, tableNames) if err != nil { log.Error().Err(err).Msg("Unable to open database") return } if *cleanup { - err = srcDb.RemoveCDC(true) + err = streamDB.RemoveCDC(true) if err != nil { log.Panic().Err(err).Msg("Unable to clean up...") } else { @@ -53,48 +45,38 @@ func main() { return } - // Set max log entries from command line - lib.MaxLogEntries = *maxLogEntries - - *metaPath = fmt.Sprintf("%s/node-%d", *metaPath, *nodeID) - raft := lib.NewRaftServer(*bindAddress, *nodeID, *metaPath, srcDb) - err = raft.Init() + rep, err := lib.NewReplicator(*nodeID, *natsAddr, *shards) if err != nil { - log.Panic().Err(err).Msg("Unable to initialize raft server") - } - - if !*followFlag { - clusterIds := makeRange(1, *shards) - err = raft.BindCluster(*initialAddrs, false, clusterIds...) - if err != nil { - log.Panic().Err(err).Msg("Unable to start Raft") - } - - log.Info().Msg("Waiting for cluster to come up...") - for uint64(len(raft.GetActiveClusters())) != *shards { - time.Sleep(500 * time.Millisecond) - } + log.Panic().Err(err).Msg("Unable to connect") } - srcDb.OnChange = onTableChanged(raft, *nodeID, *shards) + streamDB.OnChange = onTableChanged(rep, *nodeID, *shards) log.Info().Msg("Starting change data capture pipeline...") - if err := srcDb.InstallCDC(); err != nil { + if err := streamDB.InstallCDC(); err != nil { log.Error().Err(err).Msg("Unable to install change data capture pipeline") return } - err = lib.NewControlPane(raft).Run(*bindPane) + err = rep.Listen(onChangeEvent(streamDB)) if err != nil { - log.Panic().Err(err).Msg("Control pane not working") + log.Panic().Err(err).Msg("Listener terminated") } } -func onTableChanged(raft *lib.RaftServer, nodeID uint64, shards uint64) func(event *db.ChangeLogEvent) error { - return func(event *db.ChangeLogEvent) error { - if uint64(len(raft.GetActiveClusters())) != shards { - return db.ErrLogNotReadyToPublish +func onChangeEvent(streamDB *db.SqliteStreamDB) func(data []byte) error { + return func(data []byte) error { + ev := &lib.ReplicationEvent[db.ChangeLogEvent]{} + err := ev.Unmarshal(data) + if err != nil { + return err } + return streamDB.Replicate(ev.Payload) + } +} + +func onTableChanged(r *lib.Replicator, nodeID uint64, shards uint64) func(event *db.ChangeLogEvent) error { + return func(event *db.ChangeLogEvent) error { ev := &lib.ReplicationEvent[db.ChangeLogEvent]{ FromNodeId: nodeID, Payload: event, @@ -110,23 +92,11 @@ func onTableChanged(raft *lib.RaftServer, nodeID uint64, shards uint64) func(eve return err } - res, err := raft.Propose(hash, data, 1*time.Second) + err = r.Publish(hash%shards, data) if err != nil { return err } - if !res.Committed() { - return errors.New("replication failed") - } - return nil } } - -func makeRange(min, max uint64) []uint64 { - a := make([]uint64, max-min+1) - for i := range a { - a[i] = min + uint64(i) - } - return a -} From 0190fed706507a4a0544d45ebe58328fd0de0996 Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 15:39:55 -0700 Subject: [PATCH 2/9] Full working version with NATs shards --- db/change_log.go | 43 ++++++++------- lib/replication_event.go | 5 +- lib/replicator.go | 113 ++++++++++++++++++++++++++------------- marmot.go | 29 ++++++++-- 4 files changed, 128 insertions(+), 62 deletions(-) diff --git a/db/change_log.go b/db/change_log.go index caf2077..1f48248 100644 --- a/db/change_log.go +++ b/db/change_log.go @@ -66,6 +66,29 @@ func (conn *SqliteStreamDB) Replicate(event *ChangeLogEvent) error { return nil } +func (conn *SqliteStreamDB) DeleteChangeLog(event *ChangeLogEvent) (bool, error) { + metaTableName := conn.metaTable(event.TableName, changeLogName) + rs, err := conn.Delete(metaTableName). + Where(goqu.Ex{ + "state": Published, + "id": event.Id, + }). + Prepared(true). + Executor(). + Exec() + + if err != nil { + return false, err + } + + count, err := rs.RowsAffected() + if err != nil { + return false, err + } + + return count > 0, nil +} + func (conn *SqliteStreamDB) CleanupChangeLogs() (int64, error) { total := int64(0) for name := range conn.watchTablesSchema { @@ -204,25 +227,7 @@ func (conn *SqliteStreamDB) publishChangeLog() { conn.publishLock.Lock() processed := uint64(0) - // TODO: Move cleanup logic to time based cleanup - // In order to reduce frequent writes, change the logic below - // to only do in place updates, and the periodically do - // table cleanup. - defer func() { - if processed > uint64(0) { - cnt, err := conn.CleanupChangeLogs() - if err != nil { - log.Warn().Err(err).Msg("Unable to cleanup change logs") - } else if cnt > int64(0) { - log.Debug(). - Int64("cleaned", cnt). - Uint64("published", processed). - Msg("Cleaned up change log") - } - } - - conn.publishLock.Unlock() - }() + defer conn.publishLock.Unlock() for tableName := range conn.watchTablesSchema { var changes []*changeLogEntry diff --git a/lib/replication_event.go b/lib/replication_event.go index dbef5a8..47a8560 100644 --- a/lib/replication_event.go +++ b/lib/replication_event.go @@ -3,8 +3,9 @@ package lib import "github.com/fxamacker/cbor/v2" type ReplicationEvent[T any] struct { - FromNodeId uint64 - Payload *T + FromNodeId uint64 + ChangeRowId int64 + Payload *T } func (e *ReplicationEvent[T]) Marshal() ([]byte, error) { diff --git a/lib/replicator.go b/lib/replicator.go index ed9b27d..60fd59e 100644 --- a/lib/replicator.go +++ b/lib/replicator.go @@ -9,61 +9,70 @@ import ( ) const DefaultUrl = nats.DefaultURL -const MaxLogEntries = int64(1024) -const clusterPrefix = "marmot.cluster." +const NodeNamePrefix = "marmot-node" + +var MaxLogEntries = int64(1024) +var StreamNamePrefix = "marmot-changes" +var SubjectPrefix = "change-event" type Replicator struct { nodeID uint64 shards uint64 - client *nats.Conn - js nats.JetStream + client *nats.Conn + streamMap map[uint64]nats.JetStream } func NewReplicator(nodeID uint64, natsServer string, shards uint64) (*Replicator, error) { - nc, err := nats.Connect(natsServer, nats.Name(fmt.Sprintf("node-%d", nodeID))) - if err != nil { - return nil, err - } + nc, err := nats.Connect(natsServer, nats.Name(nodeName(nodeID))) - js, err := nc.JetStream() if err != nil { return nil, err } - streamCfg := &nats.StreamConfig{ - Name: "marmot", - Subjects: []string{clusterPrefix + "*"}, - Discard: nats.DiscardOld, - MaxMsgs: MaxLogEntries, - Storage: nats.FileStorage, - Retention: nats.InterestPolicy, - AllowDirect: true, - MaxConsumers: -1, - Duplicates: 0, - DenyDelete: true, - Replicas: 2, - } + streamMap := map[uint64]nats.JetStream{} + for i := uint64(0); i < shards; i++ { + js, err := nc.JetStream() + if err != nil { + return nil, err + } - info, err := js.StreamInfo("marmot") - if err != nil { - info, err = js.AddStream(streamCfg) - } else { - info, err = js.UpdateStream(streamCfg) + streamCfg := makeConfig(i+1, shards) + _, err = js.StreamInfo(streamCfg.Name) + if err != nil { + _, err = js.AddStream(streamCfg) + } else { + _, err = js.UpdateStream(streamCfg) + } + + if err != nil { + return nil, err + } + + log.Info(). + Uint64("id", i). + Msg("Created stream") + + streamMap[i+1] = js } - log.Info().Msg(fmt.Sprintf("Stream info %v", info)) return &Replicator{ client: nc, nodeID: nodeID, - js: js, - shards: shards, + + shards: shards, + streamMap: streamMap, }, nil } func (r *Replicator) Publish(hash uint64, payload []byte) error { - shard := hash % r.shards - ack, err := r.js.Publish(fmt.Sprintf("%s%d", clusterPrefix, shard), payload) + shardID := (hash % r.shards) + 1 + js, ok := r.streamMap[shardID] + if !ok { + log.Panic().Uint64("shard", shardID).Msg("Invalid shard") + } + + ack, err := js.Publish(topicName(shardID), payload) if err != nil { return err } @@ -72,8 +81,15 @@ func (r *Replicator) Publish(hash uint64, payload []byte) error { return nil } -func (r *Replicator) Listen(callback func(payload []byte) error) error { - sub, err := r.js.SubscribeSync(clusterPrefix + "*") +func (r *Replicator) Listen(shardID uint64, callback func(payload []byte) error) error { + js := r.streamMap[shardID] + + sub, err := js.SubscribeSync( + topicName(shardID), + nats.Durable(nodeName(r.nodeID)), + nats.ManualAck(), + ) + if err != nil { return err } @@ -90,8 +106,7 @@ func (r *Replicator) Listen(callback func(payload []byte) error) error { return err } - log.Debug().Str("sub", msg.Subject).Msg("Msg...") - + log.Debug().Str("sub", msg.Subject).Uint64("shard", shardID).Send() err = callback(msg.Data) if err != nil { return err @@ -105,3 +120,29 @@ func (r *Replicator) Listen(callback func(payload []byte) error) error { return nil } + +func nodeName(nodeID uint64) string { + return fmt.Sprintf("%s-%d", NodeNamePrefix, nodeID) +} + +func topicName(shardID uint64) string { + return fmt.Sprintf("%s.%d", SubjectPrefix, shardID) +} + +func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { + streamName := fmt.Sprintf("%s-%d-%d", StreamNamePrefix, totalShards, shardID) + return &nats.StreamConfig{ + Name: streamName, + Subjects: []string{topicName(shardID)}, + Discard: nats.DiscardOld, + MaxMsgs: MaxLogEntries, + Storage: nats.FileStorage, + Retention: nats.LimitsPolicy, + AllowDirect: true, + MaxConsumers: -1, + MaxMsgsPerSubject: -1, + Duplicates: 0, + DenyDelete: true, + Replicas: int(totalShards>>1 + 1), + } +} diff --git a/marmot.go b/marmot.go index 47b0a79..df8bff0 100644 --- a/marmot.go +++ b/marmot.go @@ -16,7 +16,7 @@ func main() { dbPathString := flag.String("db-path", "/tmp/marmot.db", "Path to SQLite database") nodeID := flag.Uint64("node-id", rand.Uint64(), "Node ID") natsAddr := flag.String("nats-url", lib.DefaultUrl, "NATS server URL") - shards := flag.Uint64("shards", 512, "Number of stream shards to distribute change log on") + shards := flag.Uint64("shards", 8, "Number of stream shards to distribute change log on") verbose := flag.Bool("verbose", false, "Log debug level") flag.Parse() @@ -57,9 +57,22 @@ func main() { return } - err = rep.Listen(onChangeEvent(streamDB)) + errChan := make(chan error) + for i := uint64(0); i < *shards; i++ { + go changeListener(streamDB, rep, i+1) + } + + err = <-errChan + if err != nil { + log.Panic().Err(err).Msg("Terminated listener") + } +} + +func changeListener(streamDB *db.SqliteStreamDB, rep *lib.Replicator, shard uint64) { + log.Debug().Uint64("shard", shard).Msg("Listening stream") + err := rep.Listen(shard, onChangeEvent(streamDB)) if err != nil { - log.Panic().Err(err).Msg("Listener terminated") + log.Panic().Err(err).Msg("Listener error") } } @@ -71,6 +84,11 @@ func onChangeEvent(streamDB *db.SqliteStreamDB) func(data []byte) error { return err } + ok, _ := streamDB.DeleteChangeLog(ev.Payload) + if ok { + return nil + } + return streamDB.Replicate(ev.Payload) } } @@ -78,8 +96,9 @@ func onChangeEvent(streamDB *db.SqliteStreamDB) func(data []byte) error { func onTableChanged(r *lib.Replicator, nodeID uint64, shards uint64) func(event *db.ChangeLogEvent) error { return func(event *db.ChangeLogEvent) error { ev := &lib.ReplicationEvent[db.ChangeLogEvent]{ - FromNodeId: nodeID, - Payload: event, + FromNodeId: nodeID, + ChangeRowId: event.Id, + Payload: event, } data, err := ev.Marshal() From c8fdcbe95fc1d3a4c3cf825e64004326ea18934d Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 15:40:39 -0700 Subject: [PATCH 3/9] Further cleanup --- lib/utils.go | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 lib/utils.go diff --git a/lib/utils.go b/lib/utils.go deleted file mode 100644 index 320a3f8..0000000 --- a/lib/utils.go +++ /dev/null @@ -1,28 +0,0 @@ -package lib - -import ( - "encoding/binary" - "io" -) - -func writeUint32(writer io.Writer, val uint32) error { - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, val) - _, err := writer.Write(buf) - if err != nil { - return err - } - - return nil -} - -func readUint32(reader io.Reader) (uint32, error) { - buff := make([]byte, 4, 4) - _, err := io.LimitReader(reader, 4).Read(buff) - if err != nil { - return 0, err - } - - val := binary.BigEndian.Uint32(buff) - return val, nil -} From b735aef665cec9ef60e37b4bd40b45381f4c5a52 Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 16:04:22 -0700 Subject: [PATCH 4/9] Updating README --- README.md | 61 +++++++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 56c5af0..46426ac 100644 --- a/README.md +++ b/README.md @@ -5,36 +5,34 @@ A distributed SQLite replicator. ## What is it useful for right now? If you are using SQLite as ephemeral storage, or a scenario where eventual consistency is fine for you. -Marmot can give you a solid replication between your nodes as Marmot builds on top of fault-tolerant -consensus protocol ([Multi-Raft](https://tikv.org/deep-dive/scalability/multi-raft/)), thus allowing -robust recovery and replication. This means if you are running a medium traffic website based on -SQLite you should be easily able to handle load without any problems. Read heavy workloads won't -be bottle-necked at all as Marmot serves as a side car letting you build replication cluster -without making any changes to your application code, and allows you to keep using to your -SQLite database file. In a typical setting your setup would look like this: +Marmot can give you a solid replication between your nodes as Marmot builds on top of fault-tolerant +[NATS](https://tikv.org/deep-dive/scalability/multi-raft/), thus allowing robust recovery and +replication. This means if you are running a medium traffic website based on SQLite you +should be easily able to handle load without any problems. Read heavy workloads won't +be bottle-necked at all as Marmot serves as a side car letting you build replication +cluster without making any changes to your application code, and allows you to keep +using to your SQLite database file. In a typical setting your setup would look +like this: ![image](https://user-images.githubusercontent.com/22441/190715676-8b785596-f267-49a3-aa27-21afbe74d0be.png) ## Production status -**MARMOT IS IN RELEASE CANDIDATE STAGES, WHICH MEANS MOST OF ISSUES HAVE BEEN IRONED OUT, HOWEVER THERE MIGHT BE EDGE CASE RACE CONDITIONS THAT MIGHT CAUSE PROBLEMS. FOR NOW WE RECOMMEND USAGE FOR EPHEMERAL STORAGE USE CASE ONLY WHERE LOOSING DATA, OR CORRUPTION OF DB FILE IS NOT AN ISSUE.** +**MARMOT IS READY FOR PRODUCTION USAGE NOW** -Right now it's being used for ephemeral cache storage in production services, on a very read heavy site. This easily replicates cache values across -the cluster, keeping a fast local copy of cache database. +Right now it's being used for ephemeral cache storage in production services, on a very read heavy site. +This easily replicates cache values across the cluster, keeping a fast local copy of cache database. ## Features - - MultiRaft based consensus with ability to manually move a cluster around + - Built on top of NATS - Bidirectional replication with almost masterless architecture - Ability to snapshot and fully recover from those snapshots - SQLite based log storage To be implemented for next GA: - Command batching + compression for speeding up bulk load / commit commands to propagate quickly - - On the fly join and cluster rebalancing - Per node database level command ordering - - Gossip and SRV based node discovery - - CDC output to NATS and log file ## Running @@ -46,13 +44,14 @@ go build -o build/marmot ./marmot.go Make sure you have 2 SQLite DBs with exact same schemas (ideally exact same state): ```shell -rm -rf /tmp/raft # Clear out previous raft state, only do for cold start -build/marmot -bootstrap 2@127.0.0.1:8162 -bind 127.0.0.1:8161 -bind-pane localhost:6001 -node-id 1 -db-path /tmp/cache-1.db -build/marmot -bootstrap 1@127.0.0.1:8161 -bind 127.0.0.1:8162 -bind-pane localhost:6002 -node-id 2 -db-path /tmp/cache-2.db +nats-server --jetstream +build/marmot -nats-url nats://127.0.0.1:4222 -node-id 1 -db-path /tmp/cache-1.db +build/marmot -nats-url nats://127.0.0.1:4222 -node-id 2 -db-path /tmp/cache-2.db ``` ## Demos +Demos for `v0.3.x` - [Scaling Pocketbase with Marmot](https://youtube.com/video/VSa-VJso050) - [Scaling Pocketbase with Marmot - Follow up](https://www.youtube.com/watch?v=Zapupe_FREc) @@ -63,28 +62,14 @@ configure marmot: - `cleanup` - Just cleanup and exit marmot. Useful for scenarios where you are performing a cleanup of hooks and change logs. (default: `false`) - - `db-path` - Path to DB from which given tables will be replicated. These tables can be specified in `replicate` - option. (default: `/tmp/marmot.db`) - - `replicate` - A comma seperated list of tables to replicate with no spaces in between (e.g. news,history) - (default: [empty]) **DEPRECATED after 0.3.11 now all tables are parsed and listed, this is required for - snapshots to recover quickly** + - `db-path` - Path to DB from which all tables will be replicated (default: `/tmp/marmot.db`) - `node-id` - An ID number (positive integer) to represent an ID for this node, this is required to be a unique number per node, and used for consensus protocol. (default: 0) - - `bind` - A `host:port` combination of listen for other nodes on (default: `0.0.0.0:8610`) - - `raft-path` - Path of directory to save consensus related logs, states, and snapshots (default: `/tmp/raft`) - `shards` - Number of shards over which the database tables replication will be distributed on. It serves as mechanism for - consistently hashing leader from Hash( + ) for all the nodes. These partitions can - be assigned to various nodes in cluster which allows you to distribute leadership load over multiple nodes rather - than single master. By default, there are 16 shards which means you should be easily able to have upto 16 leader - nodes. Beyond that you should use this flag to create a bigger cluster. Higher shards also mean more disk space, - and memory usage per node. Marmot has basic a very rough version of adding shards, via control pane, but it - needs more polishing to make it idiot-proof. - - `bootstrap` - A comma seperated list of initial bootstrap nodes `@:` (e.g. - `2@127.0.0.1:8162,3@127.0.0.1:8163` will specify 2 bootstrap nodes for cluster). - - `bind-pane` - A `host:port` combination for control panel address (default: `localhost:6010`). All the endpoints - are basic auth protected which should be set via `AUTH_KEY` env variable (e.g. `AUTH_KEY='Basic ...'`). This - address should not be a public accessible, and should be only used for cluster management. This in future - will serve as full control panel hosting and cluster management API. **EXPERIMENTAL** + consistently hashing JetStream from `Hash( + )`. This will allow NATS servers to + distribute load and scale for wider clusters. Look at internal docs on how these JetStreams and subjects are named. + - `nats-url` - URL string for NATS servers, it can also point to multipule servers as long as its comma separated (e.g. + `nats://user:pass@127.0.0.1:4222` or `nats://user:pass@host-a:4222, nats://user:pass@host-b:4222`) - `verbose` - Specify if system should dump debug logs on console as well. Only use this for debugging. For more details and internal workings of marmot [go to these docs](https://github.com/maxpert/marmot/blob/master/docs/overview.md). @@ -94,8 +79,8 @@ Right now there are a few limitations on current solution: - You can't watch tables selectively on a DB. This is due to various limitations around snapshot and restore mechanism. - WAL mode required - since your DB is going to be processed by multiple process the only way to have multi-process changes reliably is via WAL. - - Downloading snapshots of database is still WIP. However, if you can have start off node with same copies of DB it will - work flawlessly. + - Downloading snapshots of database is still WIP. However, it doesn't affect replication functionality as everything + is upsert or delete. Right snapshots are not restore, or initialized. - Marmot is eventually consistent. This simply means rows can get synced out of order, and `SERIALIZABLE` assumptions on transactions might not hold true anymore. From dd32ad70bdee1d680ed8b621fecbe46ef48ba6ed Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 16:12:22 -0700 Subject: [PATCH 5/9] Fixing README issues --- README.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 46426ac..e45f278 100644 --- a/README.md +++ b/README.md @@ -6,26 +6,22 @@ A distributed SQLite replicator. ## What is it useful for right now? If you are using SQLite as ephemeral storage, or a scenario where eventual consistency is fine for you. Marmot can give you a solid replication between your nodes as Marmot builds on top of fault-tolerant -[NATS](https://tikv.org/deep-dive/scalability/multi-raft/), thus allowing robust recovery and -replication. This means if you are running a medium traffic website based on SQLite you -should be easily able to handle load without any problems. Read heavy workloads won't -be bottle-necked at all as Marmot serves as a side car letting you build replication -cluster without making any changes to your application code, and allows you to keep -using to your SQLite database file. In a typical setting your setup would look -like this: - -![image](https://user-images.githubusercontent.com/22441/190715676-8b785596-f267-49a3-aa27-21afbe74d0be.png) +[NATS](https://nats.io/), thus allowing robust recovery and replication. This means if you are +running a medium traffic website based on SQLite you should be easily able to handle load +without any problems. Read heavy workloads won't be bottle-necked at all as Marmot serves +as a side car letting you build replication cluster without making any changes to your +application code, and allows you to keep using to your SQLite database file. ## Production status -**MARMOT IS READY FOR PRODUCTION USAGE NOW** +**MARMOT IS NOT READY FOR PRODUCTION USAGE** Right now it's being used for ephemeral cache storage in production services, on a very read heavy site. This easily replicates cache values across the cluster, keeping a fast local copy of cache database. ## Features - - Built on top of NATS + - Built on top of NATS, abstracting stream distribution and replication - Bidirectional replication with almost masterless architecture - Ability to snapshot and fully recover from those snapshots - SQLite based log storage @@ -51,13 +47,13 @@ build/marmot -nats-url nats://127.0.0.1:4222 -node-id 2 -db-path /tmp/cache-2.db ## Demos -Demos for `v0.3.x` +Demos for `v0.3.x` with PocketBase `v0.7.5`: - [Scaling Pocketbase with Marmot](https://youtube.com/video/VSa-VJso050) - [Scaling Pocketbase with Marmot - Follow up](https://www.youtube.com/watch?v=Zapupe_FREc) ## Documentation -Marmot is picks simplicity, and lesser knobs to configure by choice. Here are command line options you can use to +Marmot picks simplicity, and lesser knobs to configure by choice. Here are command line options you can use to configure marmot: - `cleanup` - Just cleanup and exit marmot. Useful for scenarios where you are performing a cleanup of hooks and From cff74c72364b6f2204bef3124b60a657f1a2bafa Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 16:15:34 -0700 Subject: [PATCH 6/9] Nit --- lib/replicator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/replicator.go b/lib/replicator.go index 60fd59e..95c03f8 100644 --- a/lib/replicator.go +++ b/lib/replicator.go @@ -13,7 +13,7 @@ const NodeNamePrefix = "marmot-node" var MaxLogEntries = int64(1024) var StreamNamePrefix = "marmot-changes" -var SubjectPrefix = "change-event" +var SubjectPrefix = "marmot.change" type Replicator struct { nodeID uint64 From 6a6d62f8e6d55eb4cdc270c62b904fc7a7aacc89 Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 16:29:38 -0700 Subject: [PATCH 7/9] Adding flag for custom replicas --- README.md | 1 + lib/replicator.go | 8 +++++++- marmot.go | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e45f278..73c5fa3 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ configure marmot: - `db-path` - Path to DB from which all tables will be replicated (default: `/tmp/marmot.db`) - `node-id` - An ID number (positive integer) to represent an ID for this node, this is required to be a unique number per node, and used for consensus protocol. (default: 0) + - `log-replicas` - Number of copies to be committed for single change log. By default it set to `floor(shards/2) + 1`. - `shards` - Number of shards over which the database tables replication will be distributed on. It serves as mechanism for consistently hashing JetStream from `Hash( + )`. This will allow NATS servers to distribute load and scale for wider clusters. Look at internal docs on how these JetStreams and subjects are named. diff --git a/lib/replicator.go b/lib/replicator.go index 95c03f8..45a206c 100644 --- a/lib/replicator.go +++ b/lib/replicator.go @@ -12,6 +12,7 @@ const DefaultUrl = nats.DefaultURL const NodeNamePrefix = "marmot-node" var MaxLogEntries = int64(1024) +var EntryReplicas = int(0) var StreamNamePrefix = "marmot-changes" var SubjectPrefix = "marmot.change" @@ -131,6 +132,11 @@ func topicName(shardID uint64) string { func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { streamName := fmt.Sprintf("%s-%d-%d", StreamNamePrefix, totalShards, shardID) + replicas := EntryReplicas + if replicas < 1 { + replicas = int(totalShards>>1) + 1 + } + return &nats.StreamConfig{ Name: streamName, Subjects: []string{topicName(shardID)}, @@ -143,6 +149,6 @@ func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { MaxMsgsPerSubject: -1, Duplicates: 0, DenyDelete: true, - Replicas: int(totalShards>>1 + 1), + Replicas: replicas, } } diff --git a/marmot.go b/marmot.go index df8bff0..b21beed 100644 --- a/marmot.go +++ b/marmot.go @@ -17,9 +17,12 @@ func main() { nodeID := flag.Uint64("node-id", rand.Uint64(), "Node ID") natsAddr := flag.String("nats-url", lib.DefaultUrl, "NATS server URL") shards := flag.Uint64("shards", 8, "Number of stream shards to distribute change log on") + logReplicas := flag.Int("log-replicas", 0, "Number of copies to be committed for single change log") verbose := flag.Bool("verbose", false, "Log debug level") flag.Parse() + lib.EntryReplicas = *logReplicas + if *verbose { log.Logger = log.Level(zerolog.DebugLevel) } else { @@ -28,6 +31,11 @@ func main() { log.Debug().Str("path", *dbPathString).Msg("Opening database") tableNames, err := db.GetAllDBTables(*dbPathString) + if err != nil { + log.Error().Err(err).Msg("Unable to list all tables") + return + } + streamDB, err := db.OpenStreamDB(*dbPathString, tableNames) if err != nil { log.Error().Err(err).Msg("Unable to open database") From d1e2f080bc299d29f9545d825e806e882d9bf156 Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sat, 24 Sep 2022 16:55:42 -0700 Subject: [PATCH 8/9] Sticking to - based name convention for now --- lib/replicator.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/replicator.go b/lib/replicator.go index 45a206c..4847d2e 100644 --- a/lib/replicator.go +++ b/lib/replicator.go @@ -14,7 +14,7 @@ const NodeNamePrefix = "marmot-node" var MaxLogEntries = int64(1024) var EntryReplicas = int(0) var StreamNamePrefix = "marmot-changes" -var SubjectPrefix = "marmot.change" +var SubjectPrefix = "marmot-change-log" type Replicator struct { nodeID uint64 @@ -33,12 +33,13 @@ func NewReplicator(nodeID uint64, natsServer string, shards uint64) (*Replicator streamMap := map[uint64]nats.JetStream{} for i := uint64(0); i < shards; i++ { + shard := i + 1 js, err := nc.JetStream() if err != nil { return nil, err } - streamCfg := makeConfig(i+1, shards) + streamCfg := makeShardConfig(shard, shards) _, err = js.StreamInfo(streamCfg.Name) if err != nil { _, err = js.AddStream(streamCfg) @@ -50,11 +51,11 @@ func NewReplicator(nodeID uint64, natsServer string, shards uint64) (*Replicator return nil, err } - log.Info(). - Uint64("id", i). + log.Debug(). + Uint64("shard", shard). Msg("Created stream") - streamMap[i+1] = js + streamMap[shard] = js } return &Replicator{ @@ -73,7 +74,7 @@ func (r *Replicator) Publish(hash uint64, payload []byte) error { log.Panic().Uint64("shard", shardID).Msg("Invalid shard") } - ack, err := js.Publish(topicName(shardID), payload) + ack, err := js.Publish(subjectName(shardID), payload) if err != nil { return err } @@ -86,7 +87,7 @@ func (r *Replicator) Listen(shardID uint64, callback func(payload []byte) error) js := r.streamMap[shardID] sub, err := js.SubscribeSync( - topicName(shardID), + subjectName(shardID), nats.Durable(nodeName(r.nodeID)), nats.ManualAck(), ) @@ -122,15 +123,7 @@ func (r *Replicator) Listen(shardID uint64, callback func(payload []byte) error) return nil } -func nodeName(nodeID uint64) string { - return fmt.Sprintf("%s-%d", NodeNamePrefix, nodeID) -} - -func topicName(shardID uint64) string { - return fmt.Sprintf("%s.%d", SubjectPrefix, shardID) -} - -func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { +func makeShardConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { streamName := fmt.Sprintf("%s-%d-%d", StreamNamePrefix, totalShards, shardID) replicas := EntryReplicas if replicas < 1 { @@ -139,7 +132,7 @@ func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { return &nats.StreamConfig{ Name: streamName, - Subjects: []string{topicName(shardID)}, + Subjects: []string{subjectName(shardID)}, Discard: nats.DiscardOld, MaxMsgs: MaxLogEntries, Storage: nats.FileStorage, @@ -152,3 +145,11 @@ func makeConfig(shardID uint64, totalShards uint64) *nats.StreamConfig { Replicas: replicas, } } + +func nodeName(nodeID uint64) string { + return fmt.Sprintf("%s-%d", NodeNamePrefix, nodeID) +} + +func subjectName(shardID uint64) string { + return fmt.Sprintf("%s-%d", SubjectPrefix, shardID) +} From 342708cfef08abdbb4b9bd57833c0f7c6c88f7ae Mon Sep 17 00:00:00 2001 From: Zohaib Date: Sun, 25 Sep 2022 00:47:12 -0700 Subject: [PATCH 9/9] Improving README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 73c5fa3..7f66237 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ without any problems. Read heavy workloads won't be bottle-necked at all as Marm as a side car letting you build replication cluster without making any changes to your application code, and allows you to keep using to your SQLite database file. +## Dependencies +Starting 0.4+ Marmot depends on [nats-server](https://nats.io/download/). + ## Production status **MARMOT IS NOT READY FOR PRODUCTION USAGE** @@ -46,8 +49,10 @@ build/marmot -nats-url nats://127.0.0.1:4222 -node-id 2 -db-path /tmp/cache-2.db ``` ## Demos +Demos for `v0.4.x`: + - Scaling Pocketbase with Marmot [Coming soon] -Demos for `v0.3.x` with PocketBase `v0.7.5`: +Demos for `v0.3.x` (Legacy) with PocketBase `v0.7.5`: - [Scaling Pocketbase with Marmot](https://youtube.com/video/VSa-VJso050) - [Scaling Pocketbase with Marmot - Follow up](https://www.youtube.com/watch?v=Zapupe_FREc)