Use sqlite for database, remove postgres

.example.env

@@ -1,4 +1 @@
-POSTGRES_DB=airbot
-POSTGRES_USER=airbot
-PGUSER=airbot
-POSTGRES_PASSWORD=<PASSWORD_HERE>
+SQLITE_DATA_DIR=/data/

Dockerfile

@@ -55,6 +55,10 @@ RUN adduser \
     --no-create-home \
     --uid "${UID}" \
     appuser
+
+RUN mkdir /data
+RUN chown appuser:appuser /data/
+
 USER appuser
 
 # Copy the executable from the "build" stage.

README.md

@@ -37,7 +37,6 @@ To run the bot locally:
 1. Copy `config/config_example.toml` to `config.toml` in the main directory
 1. Fill in the empty fields in `config.toml`, notably API keys and usernames
 1. Copy `.example.env` to `.env` in the main directory
-1. Set a value for `POSTGRES_PASSWORD` in `.env`
 1. Run `./start.sh`
 
 #### Documentation
@@ -55,7 +54,6 @@ To run in production (on a debian machine):
 1. Run `cd scripts`
 1. Run `./setup-vm-debian.sh` to set up the environment
 1. Fill in the empty fields in `config.toml`, notably API keys and usernames
-1. Set a value for `POSTGRES_PASSWORD` in `.env`
 1. (optional): If running in a GCE container, follow
   [these instructions](https://docs.docker.com/config/containers/logging/configure/#configure-the-default-logging-driver)
   to set your default Docker `log-driver` to `gcplogs` (in
@@ -75,41 +73,3 @@ To connect to the database's container while it's running, run `docker attach ai
 
 To disconnect from a container, press `CTRL-p CTRL-q`.
 
-#### Updating database
-
-To update Postgres to a new major version (example is moving from version **14**
-to **15**):
-
-1. Make sure the version-specific migrator image is available [here](https://github.com/tianon/docker-postgres-upgrade)
-
-1. Run:
-
-    ```shell
-    docker run --rm \
-        --env-file=.env \
-        -e POSTGRES_INITDB_ARGS="-U airbot" \
-        -v airbot_postgres-14-data:/var/lib/postgresql/14/data \
-        -v airbot_postgres-15-data:/var/lib/postgresql/15/data \
-        tianon/postgres-upgrade:14-to-15
-    ```
-
-    - Note that the `POSTGRES_INITDB_ARGS` value should match the `PGUSER` value
-      in your `.env` file.
-    - If the upgrade fails, run `docker volume rm airbot_postgres-15-data` and
-      try again.
-
-1. Run:
-
-    ```shell
-    docker run --rm \
-        -v airbot_postgres-14-data:/var/lib/postgresql/14/data \
-        -v airbot_postgres-15-data:/var/lib/postgresql/15/data \
-        busybox \
-        cp /var/lib/postgresql/14/data/pg_hba.conf /var/lib/postgresql/15/data/pg_hba.conf
-    ```
-
-1. Update `docker-compose.yml` to use the new Postgres image
-    - Under `services.database`, change `image` from `postgres:14` to `postgres:15`
-1. Update `docker-compose.yml` to use the new volume
-    - Under `volumes.postgres-data`, change `name` from `airbot_postgres-14-data`
-      to `airbot_postgres-15-data`

database/database.go

@@ -4,32 +4,40 @@ package database
 import (
 	"context"
 	"fmt"
+	"log"
 	"strings"
 
 	"github.com/airforce270/airbot/database/models"
 
-	"gorm.io/driver/postgres"
+	"github.com/glebarez/sqlite"
 	"gorm.io/gorm"
 )
 
+var pragmas = map[string]string{
+	"journal_mode": "WAL",
+	"synchronous":  "NORMAL",
+	"foreign_keys": "ON",
+
+	"user_version": "ON",
+
+	"temp_store": "2",
+	"cache_size": "-32000",
+}
+
 // Connect creates a connection to the database.
-func Connect(ctx context.Context, dbName, user, password string) (*gorm.DB, error) {
-	settings := map[string]string{
-		"host":     "database",
-		"dbname":   dbName,
-		"user":     user,
-		"password": password,
-		"port":     "5432",
-		"sslmode":  "disable",
-		"TimeZone": "UTC",
-	}
-	dsn := formatDSN(settings)
-	gormDB, err := gorm.Open(postgres.Open(dsn))
+func Connect(ctx context.Context, logger *log.Logger, dbFile string) (*gorm.DB, error) {
+	gormDB, err := gorm.Open(sqlite.Open(dbFile + formatPragmas(pragmas)))
 	if err != nil {
 		return nil, fmt.Errorf("failed to open DB connection: %w", err)
 	}
 	gormDB.WithContext(ctx)
 
+	context.AfterFunc(ctx, func() {
+		if err := close(gormDB); err != nil {
+			logger.Printf("failed to close DB cleanly: %v", err)
+		}
+	})
+
 	db, err := gormDB.DB()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get sql.DB handle: %w", err)
@@ -39,6 +47,18 @@ func Connect(ctx context.Context, dbName, user, password string) (*gorm.DB, erro
 	return gormDB, nil
 }
 
+func close(db *gorm.DB) error {
+	d, err := db.DB()
+	if err != nil {
+		return fmt.Errorf("failed to get DB handle: %w", err)
+	}
+
+	d.Exec("PRAGMA analysis_limit = 400;")
+	d.Exec("PRAGMA optimize;")
+
+	return nil
+}
+
 // Migrate performs GORM auto-migrations for all data models.
 func Migrate(db *gorm.DB) error {
 	for _, model := range models.AllModels {
@@ -68,11 +88,19 @@ func LeaveChannel(db *gorm.DB, platformName, channel string) error {
 	return nil
 }
 
-// formatDSN formats settings into a DSN for a Postgres GORM connection.
-func formatDSN(settings map[string]string) string {
-	parts := make([]string, len(settings))
-	for key, value := range settings {
-		parts = append(parts, key+"="+value)
+func formatPragmas(ps map[string]string) string {
+	var out strings.Builder
+
+	var i int
+	for p, v := range ps {
+		if i == 0 {
+			out.WriteString("?")
+		} else {
+			out.WriteString("&")
+		}
+		fmt.Fprintf(&out, "_pragma=%s(%s)", p, v)
+		i++
 	}
-	return strings.Join(parts, " ")
+
+	return out.String()
 }

database/databasetest/databasetest.go

@@ -2,19 +2,22 @@
 package databasetest
 
 import (
+	"context"
+	"log"
 	"testing"
 
 	"github.com/airforce270/airbot/database"
 	"github.com/airforce270/airbot/database/models"
 
-	"github.com/glebarez/sqlite"
 	"gorm.io/gorm"
 )
 
 // New creates a new in-memory database for testing.
 func New(t *testing.T) *gorm.DB {
 	t.Helper()
-	db, err := gorm.Open(sqlite.Open(":memory:"))
+	ctx := context.TODO()
+
+	db, err := database.Connect(ctx, log.Default(), ":memory:")
 	if err != nil {
 		t.Fatalf("Failed to create new in-memory DB: %v", err)
 	}

docker-compose.prod.yml

@@ -5,7 +5,7 @@ services:
     environment:
       - RUNNING_IN_DOCKER=true
     depends_on:
-      database:
+      cache:
         condition: service_healthy
     stdin_open: true
     tty: true

docker-compose.yml

@@ -2,19 +2,6 @@
 # https://github.com/docker/awesome-compose
 
 services:
-  database:
-    image: postgres:15
-    restart: always
-    env_file: .env
-    expose:
-      - 5432
-    volumes:
-      - postgres-data:/var/lib/postgresql/data
-    healthcheck:
-      test: [ "CMD", "pg_isready" ]
-      interval: 1s
-      timeout: 5s
-      retries: 10
   cache:
     image: valkey/valkey:7.2-alpine
     restart: always
@@ -25,6 +12,12 @@ services:
     command: valkey-server --save 30 1
     volumes:
       - redis-data:/data
+    healthcheck:
+      test: [ "CMD", "valkey-cli", "ping" ]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+
   server:
     build:
       context: .
@@ -37,13 +30,13 @@ services:
     environment:
       - RUNNING_IN_DOCKER=true
     depends_on:
-      database:
+      cache:
         condition: service_healthy
     volumes:
+      - sqlite-data:/data
       - type: bind
         source: ./config.toml
         target: /config.toml
 volumes:
-  postgres-data:
-    name: airbot_postgres-15-data
   redis-data:
+  sqlite-data:

go.mod

@@ -21,7 +21,6 @@ require (
 	golang.org/x/oauth2 v0.22.0
 	golang.org/x/sync v0.8.0
 	gonum.org/v1/gonum v0.15.0
-	gorm.io/driver/postgres v1.5.9
 	gorm.io/gorm v1.25.11
 )
 
@@ -35,10 +34,6 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
-	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.6.0 // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect

go.sum

@@ -106,14 +106,6 @@ github.com/hasura/go-graphql-client v0.13.0 h1:mPYtqToLttIk/89hIHJYQgPShdBffrloC
 github.com/hasura/go-graphql-client v0.13.0/go.mod h1:17qYcHgGSensF/wMAHKUhtMYaRZwZa3TyD7biqH9L3k=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
-github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
-github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
-github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
-github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@@ -238,10 +230,8 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
@@ -467,8 +457,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
-gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
 gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
 gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

main.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"time"
 
 	"github.com/airforce270/airbot/apiclients/supinic"
@@ -64,7 +65,7 @@ func start(ctx context.Context) (cleanup.Cleaner, postStartupResources, error) {
 	configSrc.Close()
 
 	log.Printf("Connecting to database...")
-	db, err := database.Connect(ctx, os.Getenv("POSTGRES_DB"), os.Getenv("POSTGRES_USER"), os.Getenv("POSTGRES_PASSWORD"))
+	db, err := database.Connect(ctx, log.Default(), filepath.Join(os.Getenv("SQLITE_DATA_DIR"), "sqlite.db"))
 	if err != nil {
 		return nil, postStartupResources{}, fmt.Errorf("failed to connect to database: %w", err)
 	}