feat: switch to a local postgres database for prisma (#49)

Author: zoey-kaiser

.github/workflows/ci.yaml

@@ -37,6 +37,23 @@ jobs:
 
   testCli:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    env:
+      DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432'
+      DIRECT_DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432'
+      AUTH_ORIGIN: http://localhost:3000
+      AUTH_SECRET: test123
     steps:
       - uses: actions/checkout@v4
 
@@ -74,7 +91,7 @@ jobs:
 
       # start prod-app
       - name: app:run in prod
-        run: "export AUTH_ORIGIN=http://localhost:3000 && export AUTH_SECRET=test123 && cd my-sidebase-app && npm run build && timeout 30 npm run preview || ( [[ $? -eq 124 ]] && echo \"app started and did not exit within first 30 seconds, thats good\" )"
+        run: "cd my-sidebase-app && npm run build && timeout 30 npm run preview || ( [[ $? -eq 124 ]] && echo \"app started and did not exit within first 30 seconds, thats good\" )"
 
       # start dev-app and curl from it
       - name: app:test in prod

src/configs/prisma.ts

@@ -1,5 +1,6 @@
 import { generateModuleHTMLComponent, generateModuleHTMLSnippet } from '../generators/generateModuleComponents'
 import type { ModuleConfig } from '../types'
+import { getUserPkgManager } from '../utils/getUserPkgManager'
 
 const prismaRootSchema = `// This is your Prisma schema file,
 // learn more about it in the docs: https://pris.ly/d/prisma-schema
@@ -10,11 +11,15 @@ generator client {
 }
 
 datasource db {
-  // NOTE: You probably want to change this to another database later on
-  provider = "sqlite"
+  provider = "postgres"
 
-  // This value is read from the .env file.
-  url      = env("DATABASE_URL")
+  url = env("DATABASE_URL")
+
+  // This environment variable can be the same as \`DATABASE_URL\` for non-pglite environments
+  directUrl = env("DIRECT_DATABASE_URL")
+
+  // This is required for development only.
+  shadowDatabaseUrl = "postgres://postgres@localhost/prisma-shadow?pgbouncer=true&connection_limit=1"
 }
 `
 
@@ -25,7 +30,8 @@ const prismaExampleSchema = `model Example {
 `
 
 const prismaEnvFile = `# Prisma
-DATABASE_URL=file:./db.sqlite
+DATABASE_URL="postgres://postgres@localhost:5432/postgres?pgbouncer=true&connection_limit=1"
+DIRECT_DATABASE_URL="postgres://postgres@localhost:5432/postgres?connection_limit=1"
 `
 
 const prismaExampleEndpoint = `/**
@@ -86,6 +92,84 @@ export function resetDatabase(databaseUrl?: string) {
 }
 `
 
+const pglite = `/**
+ * Script that starts a postgres database using pg-gateway (https://github.com/supabase-community/pg-gateway) and pglite (https://github.com/electric-sql/pglite).
+ *
+ * We use this database for local development with prisma ORM. The script also supports creating a \`shadow-database\`, which is a second, separate database
+ * that prisma uses for certain commands, such as \`pnpm prisma migrate dev\`: https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/shadow-database.
+ *
+ * To make use of the shadow-database add \`/prisma-shadow\` to the DSN you provide. This script will then spin up a second, in-memory-only database and connect you to it.
+ *
+ * This whole script approach is novel to us (before we used sqlite locally). Here is the PR that brought it all together: https://github.com/sidestream-tech/hanselmann-os/pull/3356
+ */
+import net from 'node:net'
+import { unlinkSync, writeFileSync } from 'node:fs'
+import { PGlite } from '@electric-sql/pglite'
+import { fromNodeSocket } from 'pg-gateway/node'
+import { join } from 'pathe'
+
+// If env var is set, we write a file to disk once the server is done starting up. This file can then be used by other processes to check whether the database is ready
+const doWriteHealthFile = process.env.WRITE_HEALTH_FILE === 'true'
+const HEALTH_FILE_NAME = 'pgliteHealthz'
+
+const db = new PGlite({ dataDir: join(import.meta.dirname, 'pglite-data') })
+let activeDb = db
+
+const server = net.createServer(async (socket) => {
+  activeDb = db
+
+  console.info(\`Client connected: \${socket.remoteAddress}:\${socket.remotePort}\`)
+  await fromNodeSocket(socket, {
+    serverVersion: '16.3',
+
+    auth: {
+      // No password required
+      method: 'trust',
+    },
+
+    async onStartup({ clientParams }) {
+      // create a temp in-memory instance if connecting to the prisma shadow DB
+      if (clientParams?.database === 'prisma-shadow') {
+        console.info('Connecting client to shadow database')
+        activeDb = new PGlite()
+      }
+
+      // Wait for PGlite to be ready before further processing
+      await activeDb.waitReady
+    },
+
+    // Hook into each client message
+    async onMessage(data, { isAuthenticated }) {
+      // Only forward messages to PGlite after authentication
+      if (!isAuthenticated) {
+        return
+      }
+
+      // Forward raw message to PGlite and send response to client
+      return await activeDb.execProtocolRaw(data)
+    },
+  })
+
+  socket.on('end', () => {
+    console.info('Client disconnected')
+  })
+})
+
+server.listen(5432, () => {
+  if (doWriteHealthFile) {
+    writeFileSync(HEALTH_FILE_NAME, '')
+  }
+
+  console.info('Server listening on port 5432')
+})
+
+server.on('close', () => {
+  if (doWriteHealthFile) {
+    unlinkSync(HEALTH_FILE_NAME)
+  }
+})
+`
+
 const prismaDemoComponent = `<script lang="ts" setup>
 const { data: examples } = useFetch('/api/examples')
 </script>
@@ -106,7 +190,12 @@ const { data: examples } = useFetch('/api/examples')
 const prisma: ModuleConfig = {
   humanReadableName: 'Prisma ORM',
   description: 'Next-generation Node.js and TypeScript ORM. See more: https://www.prisma.io/',
-  scripts: [],
+  scripts: [
+    {
+      name: 'db',
+      command: 'vite-node prisma/pglite.ts',
+    }
+  ],
   dependencies: [
     {
       name: 'prisma',
@@ -117,6 +206,21 @@ const prisma: ModuleConfig = {
       name: '@prisma/client',
       version: '^5.18.0',
       isDev: false
+    },
+    {
+      name: '@electric-sql/pglite',
+      version: '^0.2.9',
+      isDev: true,
+    },
+    {
+      name: 'pg-gateway',
+      version: '0.3.0-beta.3',
+      isDev: true,
+    },
+    {
+      name: 'vite-node',
+      version: '^2.1.1',
+      isDev: true,
     }
   ],
   nuxtConfig: {},
@@ -141,10 +245,15 @@ const prisma: ModuleConfig = {
   }, {
     path: 'components/Welcome/PrismaDemo.vue',
     content: prismaDemoComponent,
+  }, {
+    path: 'prisma/pglite.ts',
+    content: pglite,
   }],
   tasksPostInstall: [
     '- [ ] Prisma: Edit your `prisma/prisma.schema` to your liking',
+    `- [ ] Prisma: Start your local postgres database using \`${getUserPkgManager()} run db\``,
     '- [ ] Prisma: Run `npx prisma db push` to sync the schema to your database & generate the Prisma Client',
+    '- [ ] Prisma: Add `**/*/pglite-data` and `pgliteHealthz` to your `.gitignore` file'
   ],
   indexVue: generateModuleHTMLSnippet('WelcomePrismaDemo'),
 }

src/messages/goodbye.ts

@@ -30,6 +30,7 @@ export function sayGoodbye(preferences: Preferences) {
   }
 
   if (preferences.addModules?.includes('prisma') || preferences.setStack === 'cheviot') {
+    sayCommand(`${packageManager} run db`, 'Start the local postgres database in a new window')
     sayCommand('npx prisma db push', 'Initialize the database & Prisma client')
   }