feat: docs stored as embeddings for ai (#3511)

* feat: create database embeddings of the docsite for search and ai

* chore: pr feedback

.github/workflows/on_merge_to_main_supabase.yml

@@ -0,0 +1,60 @@
+name: Deploy Supabase Migrations to Production
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    env:
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.SUPABASE_PRODUCTION_DB_PASSWORD }}
+      SUPABASE_PROJECT_ID: ${{ secrets.SUPABASE_PRODUCTION_PROJECT_ID }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js 20.5
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.5.x
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT
+
+      - name: Load Yarn cache
+        uses: actions/cache@v3
+        id: yarn_cache_id
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn
+
+      - name: Node modules cache
+        uses: actions/cache@v3
+        id: node_modules_cache_id
+        with:
+          path: |
+            node_modules
+            */*/node_modules
+          key: ${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install Dependencies
+        if: steps.yarn_cache_id.outputs.cache-hit != 'true' || steps.node_modules_cache_id.outputs.cache-hit != 'true'
+        run: yarn install --immutable
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - run: yarn workspace @twilio-paste/backend supabase link --project-ref $SUPABASE_PROJECT_ID
+      - run: yarn workspace @twilio-paste/backend supabase db push

.github/workflows/on_pull_request_supabase.yml

@@ -0,0 +1,62 @@
+name: Supabase PR Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js 20.5
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.5.x
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT
+
+      - name: Load Yarn cache
+        uses: actions/cache@v3
+        id: yarn_cache_id
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn
+
+      - name: Node modules cache
+        uses: actions/cache@v3
+        id: node_modules_cache_id
+        with:
+          path: |
+            node_modules
+            */*/node_modules
+          key: ${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install Dependencies
+        if: steps.yarn_cache_id.outputs.cache-hit != 'true' || steps.node_modules_cache_id.outputs.cache-hit != 'true'
+        run: yarn install --immutable
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - name: Start Supabase local development setup
+        run: yarn start:backend:dev
+
+      - name: Verify generated types are checked in
+        run: |
+          yarn generate:db-types
+          if ! git diff --ignore-space-at-eol --exit-code --quiet apps/backend/supabase/schema.gen.ts; then
+            echo "Detected uncommitted changes after build. See status below:"
+            git diff
+            exit 1
+          fi

.github/workflows/update_docs_embed.yml

@@ -0,0 +1,77 @@
+name: Generate Embeddings for AI Search
+
+on:
+  workflow_dispatch:
+    inputs:
+      refresh:
+        description: "Refresh all pages"
+        required: false
+        type: boolean
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    env:
+      OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      PROJECT_ID: ${{ secrets.SUPABASE_PRODUCTION_PROJECT_ID }}
+      GH_SERVICE_ACC_DISCUSSIONS_TOKEN: ${{ secrets.GH_SERVICE_ACC_DISCUSSIONS_TOKEN }}
+      SUPABASE_URL: ${{ secrets.SUPABASE_PRODUCTION_URL }}
+      SUPABASE_KEY: ${{ secrets.SUPABASE_PRODUCTION_KEY }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.SUPABASE_PRODUCTION_DB_PASSWORD }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: nrwl/nx-set-shas@v3
+
+      - name: Setup Node.js 20.5
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.5.x
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT
+
+      - name: Load Yarn cache
+        uses: actions/cache@v3
+        id: yarn_cache_id
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn
+
+      - name: Node modules cache
+        uses: actions/cache@v3
+        id: node_modules_cache_id
+        with:
+          path: |
+            node_modules
+            */*/node_modules
+          key: ${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install Dependencies
+        if: steps.yarn_cache_id.outputs.cache-hit != 'true' || steps.node_modules_cache_id.outputs.cache-hit != 'true'
+        run: yarn install --immutable
+
+      - name: Link Supabase project
+        run: yarn workspace @twilio-paste/backend supabase link --project-ref $PROJECT_ID
+
+      - name: Run migrations
+        run: yarn workspace @twilio-paste/backend supabase db push
+
+      - name: Update embeddings
+        if: ${{ !inputs.refresh }}
+        run: yarn generate:embeddings
+
+      - name: Refresh embeddings
+        if: ${{ inputs.refresh }}
+        run: yarn generate:embeddings:refresh

.prettierignore

@@ -29,3 +29,4 @@ tsconfig.build.tsbuildinfo
 build.icon-list.js
 
 **/*.hbs
+apps/backend/supabase/schema.gen.ts

apps/backend/README.md

@@ -0,0 +1,39 @@
+# Backend
+
+Uses Supabase. You need docker running locally to start local dev. You only need to do that when editing.
+
+## Handy scripts:
+
+Note: all scripts assume the backend directory is the current working directory.
+
+Generating migrations when you've edited the DB in the Supabase console.
+
+```
+yarn supabase db diff --use-migra name_of_migration -f name_of_migration
+```
+
+Reset db after migration to test migration files. Migration files are generated by the supabase db when you make changes and run the above command.
+
+https://supabase.com/docs/guides/cli/managing-environments?environment=production#create-a-new-migration
+
+```
+yarn supabase db reset
+```
+
+Link local to upstream project so you can push migrations to staging or prod projects
+
+```
+yarn supabase link --project-ref $PROJECT_REF
+```
+
+Push migrations to upstream / clooud project such as staging
+
+```
+yarn supabase db push
+```
+
+Generate db types after migrations (from root of monorepo)
+
+```
+yarn generate:db-types
+```

apps/backend/backend.code-workspace

@@ -0,0 +1,17 @@
+{
+  "folders": [
+    {
+      "name": "project-root",
+      "path": "./"
+    },
+    {
+      "name": "supabase-functions",
+      "path": "supabase/functions"
+    }
+  ],
+  "settings": {
+    "files.exclude": {
+      "supabase/functions/": true
+    }
+  }
+}

apps/backend/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@twilio-paste/backend",
+  "private": true,
+  "license": "MIT",
+  "scripts": {
+    "start": "yarn supabase start",
+    "stop": "yarn supabase stop",
+    "generate:db-types": "yarn supabase gen types typescript --local > supabase/schema.gen.ts"
+  },
+  "devDependencies": {
+    "supabase": "^1.99.5"
+  }
+}

apps/backend/supabase/.gitignore

@@ -0,0 +1,3 @@
+# Supabase
+.branches
+.temp