Merge pull request #317 from bmeg/feature/sqlite

Adds support for sqlite
bmeg · Oct 3, 2024 · b163abb · b163abb
2 parents 99c314b + 529684b
commit b163abb
Show file tree

Hide file tree

Showing 14 changed files with 1,334 additions and 66 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -13,18 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: ^1.18
+          go-version: ^1.22.6
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build
         run: go build -v ./
 
       - name: Store grip
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: gripBin
           path: grip
@@ -35,28 +35,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: ^1.18
+          go-version: ^1.22.6
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: run unit tests
         run: |
           go test ./test/... -config badger.yml
-  
-
+
 
   badgerTest:
     needs: build
     name: Badger Conformance
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Badger Test
@@ -65,19 +64,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/badger.yml &
           sleep 5
           make test-conformance
-  
+
 
   pebbleTest:
     needs: build
     name: Pebble Conformance
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Pebble Test
@@ -86,19 +85,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/pebble.yml &
           sleep 5
           make test-conformance
-    
+
 
   mongoTest:
     needs: build
     name: Mongo Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Mongo Conformance
@@ -108,41 +107,41 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo.yml &
           sleep 5
           make test-conformance
-  
+
 
   mongoCoreTest:
     needs: build
     name: Mongo Core Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
-      - name: Mongo Conformance
+      - name: Mongo Core Conformance
         run: |
           chmod +x grip
           make start-mongo
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo-core-processor.yml &
           sleep 5
           make test-conformance
-  
+
 
   elasticTest:
     needs: build
     name: Elastic Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Elastic Conformance
@@ -153,19 +152,19 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/elastic.yml &
           sleep 5
           make test-conformance
-  
 
-  portgresTest:
+
+  postgresTest:
     needs: build
-    name: Portgres Test
+    name: Postgres Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Postgres Conformance
@@ -176,21 +175,42 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/psql.yml &
           sleep 5
           python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
-  
+
+
+  sqliteTest:
+    needs: build
+    name: Sqlite Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Python Dependencies for Conformance
+        run: pip install requests numpy
+      - name: Download grip
+        uses: actions/download-artifact@v4
+        with:
+          name: gripBin
+      - name: Sqlite Conformance
+        run: |
+          chmod +x grip
+          ./grip server --rpc-port 18202 --http-port 18201 --config ./test/sqlite.yml &
+          sleep 5
+          python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
+
 
   gripperTest:
     needs: build
     name: Gripper Test
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Update pip
         run: pip install --upgrade pip
       - name: Python Dependencies for Conformance
         run: pip install -U requests numpy grpcio-tools protobuf
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Gripper Conformance
@@ -210,11 +230,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Python Dependencies for Conformance
         run: pip install requests numpy PyYAML
       - name: Download grip
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: gripBin
       - name: Auth Test
@@ -224,30 +244,29 @@ jobs:
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/badger-auth.yml &
           sleep 5
           # simple auth
-          # run tests without credentials, should fail 
+          # run tests without credentials, should fail
           if make test-conformance
           then
             echo "ERROR: Conformance tests ran without credentials." ; exit 1
           else
-            echo "Got expected auth error" 
-          fi          
+            echo "Got expected auth error"
+          fi
           # run specialized role based tests
           make test-authorization ARGS="--grip_config_file_path test/badger-auth.yml"
-  
-
-
+
+
 
     #gridsTest:
     #  needs: build
     #  name: GRIDs Conformance
     #  runs-on: ubuntu-latest
     #  steps:
     #  - name: Check out code
-    #    uses: actions/checkout@v2
+    #    uses: actions/checkout@v4
     #  - name: Python Dependencies for Conformance
     #    run: pip install requests numpy
     #  - name: Download grip
-    #    uses: actions/download-artifact@v2
+    #    uses: actions/download-artifact@v4
     #    with:
     #      name: gripBin
     #  - name: GRIDs unit tests

diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -93,6 +93,8 @@ var Cmd = &cobra.Command{
 				dconf.AddMongoDefault()
 			} else if driver == "grids" {
 				dconf.AddGridsDefault()
+			} else if driver == "sqlite" {
+				dconf.AddSqliteDefault()
 			}
 		}
 		if pluginDir != "" {

diff --git a/config/config.go b/config/config.go
@@ -15,6 +15,7 @@ import (
 	"github.com/bmeg/grip/log"
 	"github.com/bmeg/grip/mongo"
 	"github.com/bmeg/grip/psql"
+	"github.com/bmeg/grip/sqlite"
 	"github.com/bmeg/grip/util"
 	"github.com/bmeg/grip/util/duration"
 	"github.com/bmeg/grip/util/rpc"
@@ -35,6 +36,7 @@ type DriverConfig struct {
 	MongoDB       *mongo.Config
 	PSQL          *psql.Config
 	ExistingSQL   *esql.Config
+	Sqlite        *sqlite.Config
 	Gripper       *gripper.Config
 }
 
@@ -101,6 +103,12 @@ func (conf *Config) AddMongoDefault() {
 	conf.Default = "mongo"
 }
 
+func (conf *Config) AddSqliteDefault() {
+	c := sqlite.Config{DBName: "grip-sqlite.db"}
+	conf.Drivers["sqlite"] = DriverConfig{Sqlite: &c}
+	conf.Default = "sqlite"
+}
+
 func (conf *Config) AddGridsDefault() {
 	n := "grip-grids.db"
 	conf.Drivers["grids"] = DriverConfig{Grids: &n}
@@ -130,6 +138,9 @@ func TestifyConfig(c *Config) {
 		d.Elasticsearch.DBName = "gripdb-" + rand
 		d.Elasticsearch.Synchronous = true
 	}
+	if d.Sqlite != nil {
+		d.Sqlite.DBName = "gripdb-" + rand
+	}
 	c.Drivers[c.Default] = d
 }
 

diff --git a/go.mod b/go.mod
@@ -32,6 +32,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/machinebox/graphql v0.2.2
+	github.com/mattn/go-sqlite3 v1.14.23
 	github.com/minio/minio-go/v7 v7.0.73
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/mongodb/mongo-tools v0.0.0-20240715143021-aa6a140d3f17

diff --git a/go.sum b/go.sum
@@ -276,8 +276,9 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0=
+github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=