chore: set up some logging and rollbar error handling for ai endpoints (

#3582) * chore: set up some logging and rollbar error handling for ai endpoints * fix: logging * chore: update rollbar * chore: next 14 * chore: remove rollbar from ai streaming * chore: check rewust body and make sure biome formats files better * chore: create search tracking * chore: run migrations on previews
twilio-labs · Oct 30, 2023 · 701995f · 701995f
1 parent 2059b17
commit 701995f
Show file tree

Hide file tree

Showing 13 changed files with 381 additions and 255 deletions.
diff --git a/.github/workflows/on_pull_request_supabase.yml b/.github/workflows/on_pull_request_supabase.yml
@@ -8,6 +8,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    env:
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.SUPABASE_STAGING_DB_PASSWORD }}
+      SUPABASE_PROJECT_ID: ${{ secrets.SUPABASE_STAGING_PROJECT_ID }}
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -49,6 +54,12 @@ jobs:
         with:
           version: latest
 
+      - name: Link Supabase project
+        run: yarn workspace @twilio-paste/backend supabase link --project-ref $SUPABASE_PROJECT_ID
+
+      - name: Run migrations on staging
+        run: yarn workspace @twilio-paste/backend supabase db push
+
       - name: Start Supabase local development setup
         run: yarn start:backend:dev
 

diff --git a/apps/backend/supabase/migrations/20231027224426_track-search-usage.sql b/apps/backend/supabase/migrations/20231027224426_track-search-usage.sql
@@ -0,0 +1,53 @@
+create table "public"."queries" (
+    "id" bigint generated by default as identity not null,
+    "created_at" timestamp with time zone not null default now(),
+    "query_string" character varying,
+    "type" text
+);
+
+
+alter table "public"."queries" enable row level security;
+
+alter table "public"."page" enable row level security;
+
+alter table "public"."page_section" enable row level security;
+
+CREATE UNIQUE INDEX queries_pkey ON public.queries USING btree (id);
+
+alter table "public"."queries" add constraint "queries_pkey" PRIMARY KEY using index "queries_pkey";
+
+create policy "Public access"
+on "public"."page"
+as permissive
+for all
+to public
+using (true)
+with check (true);
+
+
+create policy "Public access"
+on "public"."page_section"
+as permissive
+for all
+to public
+using (true)
+with check (true);
+
+
+create policy "Enable read access for all"
+on "public"."queries"
+as permissive
+for select
+to public
+using (true);
+
+
+create policy "Enable write access for all"
+on "public"."queries"
+as permissive
+for insert
+to public
+with check (true);
+
+
+
diff --git a/apps/backend/supabase/schema.gen.ts b/apps/backend/supabase/schema.gen.ts
@@ -114,6 +114,27 @@ export interface Database {
           }
         ]
       }
+      queries: {
+        Row: {
+          created_at: string
+          id: number
+          query_string: string | null
+          type: string | null
+        }
+        Insert: {
+          created_at?: string
+          id?: number
+          query_string?: string | null
+          type?: string | null
+        }
+        Update: {
+          created_at?: string
+          id?: number
+          query_string?: string | null
+          type?: string | null
+        }
+        Relationships: []
+      }
     }
     Views: {
       [_ in never]: never

diff --git a/biome.json b/biome.json
@@ -3,25 +3,25 @@
   "files": {
     "maxSize": 5242880,
     "ignore": [
-      "node_modules",
-      "bower_components",
-      "dist",
-      "bin",
-      "docs",
-      ".temp",
-      "__testfixtures__",
-      "__fixtures__",
+      "node_modules/",
+      "bower_components/",
+      "dist/",
+      "bin/",
+      "docs/",
+      ".temp/",
+      "__testfixtures__/",
+      "__fixtures__/",
       "packages/paste-icons/cjs",
       "packages/paste-icons/esm",
       "packages/paste-icons/json",
       "packages/paste-theme-designer/public",
       "packages/paste-theme-designer/out",
       "packages/paste-token-contrast-checker/public",
       "packages/paste-website/data",
-      ".cache",
-      ".next",
-      ".netlify",
-      ".yarn",
+      ".cache/",
+      ".next/",
+      ".netlify/",
+      ".yarn/",
       "packages/**/dist/*",
       "tsconfig.build.tsbuildinfo",
       "**/*.d.ts",

diff --git a/packages/paste-theme-designer/package.json b/packages/paste-theme-designer/package.json
@@ -16,7 +16,7 @@
     "@datadog/browser-rum": "^4.46.0",
     "@twilio-paste/core": "^20.0.0",
     "@twilio-paste/icons": "^12.0.0",
-    "next": "^13.1.6",
+    "next": "^14.0.0",
     "react": "^18.0.0",
     "react-color": "^2.19.3"
   }

diff --git a/packages/paste-website/next.config.mjs b/packages/paste-website/next.config.mjs
@@ -25,9 +25,6 @@ const nextConfig = {
   compiler: {
     emotion: true,
   },
-  experimental: {
-    legacyBrowsers: false,
-  },
   // https://nextjs.org/docs/pages/api-reference/next-config-js/headers
   async headers() {
     return [

diff --git a/packages/paste-website/package.json b/packages/paste-website/package.json
@@ -24,8 +24,8 @@
     "@mdx-js/loader": "^1.6.22",
     "@mdx-js/mdx": "^1.6.22",
     "@mdx-js/react": "^1.6.22",
-    "@next/bundle-analyzer": "^13.1.6",
-    "@next/mdx": "^13.1.6",
+    "@next/bundle-analyzer": "^14.0.0",
+    "@next/mdx": "^14.0.0",
     "@octokit/core": "^5.0.1",
     "@octokit/plugin-paginate-graphql": "^4.0.0",
     "@sparticuz/chromium": "^110.0.0",
@@ -157,7 +157,7 @@
     "mdast-util-to-string": "^3.1.1",
     "micromark-extension-mdxjs": "^2.0.0",
     "minimist": "^1.2.8",
-    "next": "^13.1.6",
+    "next": "^14.0.0",
     "openai": "^4.10.0",
     "openai-edge": "^1.2.2",
     "pretty-format": "^28.1.0",
@@ -173,18 +173,18 @@
     "react-scrollspy": "^3.4.0",
     "react-visibility-sensor": "5.1.1",
     "remark-gfm": "^3.0.1",
-    "rollbar": "^2.25.0",
+    "rollbar": "^2.26.2",
     "sharp": "^0.32.5",
     "typescript": "^4.9.4",
     "unist-builder": "^4.0.0",
     "unist-util-filter": "^5.0.1",
     "unist-util-visit-esm": "npm:unist-util-visit@^4.1.2",
     "use-resize-observer": "^9.1.0",
     "uuid": "^9.0.1",
-    "winston": "^3.8.1"
+    "winston": "^3.11.0"
   },
   "devDependencies": {
-    "@next/eslint-plugin-next": "^13.1.6",
+    "@next/eslint-plugin-next": "^14.0.0",
     "@storybook/react": "7.0.6",
     "@testing-library/react": "^13.4.0",
     "tsx": "^3.12.10"

diff --git a/packages/paste-website/src/pages/api/ai.ts b/packages/paste-website/src/pages/api/ai.ts
@@ -44,9 +44,18 @@ const config = new Configuration({
 });
 const openai = new OpenAIApi(config);
 
+/**
+ * Because we're using an edge function for streaming we can't use winston for logging
+ * or rollbar for error reporting. Instead we'll use console.log and console.error and
+ * Datadog synthetics to monitor up time.
+ */
 export const runtime = "edge";
 
+const LOG_PREFIX = "[/api/ai]:";
+
 export default async function handler(req: NextRequest): Promise<void | Response> {
+  // eslint-disable-next-line no-console
+  console.log(`${LOG_PREFIX} Incoming request`);
   try {
     if (!openAiKey) {
       throw new ApplicationError("Missing environment variable OPENAI_API_KEY");
@@ -64,12 +73,16 @@ export default async function handler(req: NextRequest): Promise<void | Response
     }
 
     const requestData = await req.json();
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Request data`, { requestData });
 
     if (!requestData) {
       throw new UserError("Missing request data");
     }
 
     const { prompt: query, secret } = requestData;
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} User query`, { query });
 
     if (!secret || secret !== openAiSecret) {
       throw new UserError("Incorrect 'secret' in request data");
@@ -81,8 +94,14 @@ export default async function handler(req: NextRequest): Promise<void | Response
 
     const supabaseClient = createClient(supabaseUrl, supabaseServiceKey);
 
-    // Moderate the content to comply with OpenAI T&C
     const sanitizedQuery = query.trim();
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Sanitized query`, { sanitizedQuery });
+
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Moderate user prompt`);
+
+    // Moderate the content to comply with OpenAI T&C
     const moderationResponse: CreateModerationResponse = await openai
       .createModeration({ input: sanitizedQuery })
       .then((res: any) => res.json());
@@ -93,6 +112,8 @@ export default async function handler(req: NextRequest): Promise<void | Response
       throw new ApplicationError("Failed to moderate content", moderationResponse.error.message);
     }
     const [results] = moderationResponse.results;
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Moderated prompt`, { results });
 
     if (results.flagged) {
       throw new UserError("Flagged content", {
@@ -101,6 +122,9 @@ export default async function handler(req: NextRequest): Promise<void | Response
       });
     }
 
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Reqesting openai embedding`);
+
     // Create embedding from query
     const embeddingResponse = await openai.createEmbedding({
       model: "text-embedding-ada-002",
@@ -115,6 +139,9 @@ export default async function handler(req: NextRequest): Promise<void | Response
       data: [{ embedding }],
     }: CreateEmbeddingResponse = await embeddingResponse.json();
 
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Request Page sections based on embeddings`);
+
     const { error: matchError, data: pageSections } = await supabaseClient.rpc("match_page_sections_for_ai", {
       embedding,
       /* eslint-disable camelcase */
@@ -128,6 +155,9 @@ export default async function handler(req: NextRequest): Promise<void | Response
       throw new ApplicationError("Failed to match page sections", matchError);
     }
 
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Returned ${pageSections.length} page sections`);
+
     const tokenizer = new GPT3Tokenizer({ type: "gpt3" });
     let tokenCount = 0;
     let contextText = "";
@@ -144,6 +174,9 @@ export default async function handler(req: NextRequest): Promise<void | Response
       contextText += `${content.trim()}\n---\n`;
     }
 
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Context text: ${contextText}`);
+
     const prompt = codeBlock`
       ${oneLine`
         Your name is PasteBot. You are a very enthusiastic Paste design system
@@ -168,6 +201,8 @@ export default async function handler(req: NextRequest): Promise<void | Response
       role: "user",
       content: prompt,
     };
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Request chat completion`);
 
     const response = await openai.createChatCompletion({
       model: "gpt-4",
@@ -183,13 +218,19 @@ export default async function handler(req: NextRequest): Promise<void | Response
       throw new ApplicationError("Failed to generate completion", error);
     }
 
+    // eslint-disable-next-line no-console
+    console.log(`${LOG_PREFIX} Open ai Returned response`);
+
     // Transform the response into a readable stream
     const stream = OpenAIStream(response);
 
     // Return a StreamingTextResponse, which can be consumed by the client
     return new StreamingTextResponse(stream);
   } catch (error: unknown) {
     if (error instanceof UserError) {
+      // eslint-disable-next-line no-console
+      console.error(`${LOG_PREFIX} User error`, { error });
+
       return new Response(
         JSON.stringify({
           error: error.message,
@@ -203,11 +244,11 @@ export default async function handler(req: NextRequest): Promise<void | Response
     } else if (error instanceof ApplicationError) {
       // Print out application errors with their additional data
       // eslint-disable-next-line no-console
-      console.error(`${error.message}: ${JSON.stringify(error.data)}`);
+      console.error(`${LOG_PREFIX} ${error.message}: ${JSON.stringify(error.data)}`);
     } else {
       // Print out unexpected errors as is to help with debugging
       // eslint-disable-next-line no-console
-      console.error(error);
+      console.error(`${LOG_PREFIX} ${error}`);
     }
 
     return new Response(