Connect chat feedback

.github/workflows/deploy-dev.yml

@@ -25,6 +25,7 @@ jobs:
             | sed s/{CODA_INCOMING_TOKEN}/${{ secrets.CODA_INCOMING_TOKEN }}/ \
             | sed s/{CODA_WRITES_TOKEN}/${{ secrets.CODA_WRITES_TOKEN }}/ \
             | sed s/{GOOGLE_ANALYTICS_ID}/${{ secrets.GOOGLE_ANALYTICS_ID }}/ \
+            | sed s/{DISCORD_LOGGING_URL}/${{ secrets.DISCORD_LOGGING_URL }}/ \
             > wrangler.toml
           npm ci
           npm run deploy

.github/workflows/deploy.yml

@@ -24,6 +24,7 @@ jobs:
             | sed s/{CODA_INCOMING_TOKEN}/${{ secrets.CODA_INCOMING_TOKEN }}/ \
             | sed s/{CODA_WRITES_TOKEN}/${{ secrets.CODA_WRITES_TOKEN }}/ \
             | sed s/{GOOGLE_ANALYTICS_ID}/${{ secrets.GOOGLE_ANALYTICS_ID }}/ \
+            | sed s/{DISCORD_LOGGING_URL}/${{ secrets.DISCORD_LOGGING_URL }}/ \
             > wrangler.toml
           npm ci
           npm run deploy

app/components/Chatbot/ChatEntry.tsx

@@ -3,7 +3,7 @@ import {Link} from '@remix-run/react'
 import MarkdownIt from 'markdown-it'
 import QuestionMarkIcon from '~/components/icons-generated/QuestionMark'
 import Contents from '~/components/Article/Contents'
-import Feedback from '~/components/Feedback'
+import Feedback, {logFeedback} from '~/components/Feedback'
 import useGlossary from '~/hooks/useGlossary'
 import './chat_entry.css'
 import type {Entry, AssistantEntry, StampyEntry, Citation, ErrorMessage} from '~/hooks/useChat'
@@ -136,7 +136,7 @@ const Reference = (citation: Citation) => {
   )
 }
 
-const ChatbotReply = ({phase, content, citationsMap}: AssistantEntry) => {
+const ChatbotReply = ({question, phase, content, citationsMap}: AssistantEntry) => {
   const citations = [] as Citation[]
   citationsMap?.forEach((v) => {
     citations.push(v)
@@ -193,6 +193,9 @@ const ChatbotReply = ({phase, content, citationsMap}: AssistantEntry) => {
           pageid="chatbot"
           upHint="This response was helpful"
           downHint="This response was unhelpful"
+          onSubmit={async (message: string, option?: string) =>
+            logFeedback({message, option, type: 'bot', question, answer: content, citations})
+          }
           options={[
             'Making things up',
             'Wrong subject',
@@ -227,6 +230,9 @@ const StampyArticle = ({pageid, content, title}: StampyEntry) => {
           pageid={pageid}
           upHint="This response was helpful"
           downHint="This response was unhelpful"
+          onSubmit={async (message: string, option?: string) =>
+            logFeedback({message, option, type: 'human', question: title, answer: content, pageid})
+          }
           options={[
             'Making things up',
             'Wrong subject',

app/components/Chatbot/index.tsx

@@ -246,6 +246,7 @@ export const Chatbot = ({question, questions, settings}: ChatbotProps) => {
 
     // Add a new history entry, replacing the previous one if it was canceled
     const message = {content: question, role: 'user'} as Entry
+    const answer = {role: 'assistant', question} as AssistantEntry
     setHistory((current) => {
       const last = current[current.length - 1]
       if (
@@ -254,15 +255,11 @@ export const Chatbot = ({question, questions, settings}: ChatbotProps) => {
         (last?.role === 'stampy' && last?.content) ||
         ['error'].includes(last?.role)
       ) {
-        return [...current, message, {role: 'assistant'} as AssistantEntry]
+        return [...current, message, answer]
       } else if (last?.role === 'user' && last?.content === question) {
-        return [...current.slice(0, current.length - 1), {role: 'assistant'} as AssistantEntry]
+        return [...current.slice(0, current.length - 1), answer]
       }
-      return [
-        ...current.slice(0, current.length - 2),
-        message,
-        {role: 'assistant'} as AssistantEntry,
-      ]
+      return [...current.slice(0, current.length - 2), message, answer]
     })
 
     setFollowups(undefined)

app/components/Feedback/Form.tsx

@@ -4,11 +4,13 @@ import useOutsideOnClick from '~/hooks/useOnOutsideClick'
 import './feedback.css'
 
 export type FeedbackFormProps = {
+  onSubmit?: (msg: string, option?: string) => Promise<any>
   onClose?: () => void
   options?: string[]
 }
-const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
+const FeedbackForm = ({onSubmit, onClose, options}: FeedbackFormProps) => {
   const [selected, setSelected] = useState<string>()
+  const [message, setMessage] = useState('')
   const [enabledSubmit, setEnabledSubmit] = useState(!options)
   const [numClicks, setNumClicks] = useState(0)
   const clickCheckerRef = useOutsideOnClick(onClose)
@@ -28,7 +30,8 @@ const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
     setEnabledSubmit(true)
   }
 
-  const handleSubmit = () => {
+  const handleSubmit = async () => {
+    onSubmit && (await onSubmit(message, selected))
     onClose && onClose()
   }
 
@@ -56,6 +59,7 @@ const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
         name="feedback-text"
         className={['feedback-text bordered', !options ? 'no-options' : ''].join(' ')}
         placeholder="Leave a comment (optional)"
+        onChange={(e) => setMessage(e.target.value)}
       />
       <Button className="primary full-width" action={handleSubmit} disabled={!enabledSubmit}>
         <p>Submit feedback</p>

app/components/Feedback/index.tsx

@@ -3,6 +3,25 @@ import {CompositeButton} from '~/components/Button'
 import {Action, ActionType} from '~/routes/questions.actions'
 import './feedback.css'
 import FeedbackForm from './Form'
+import type {Citation} from '~/hooks/useChat'
+
+type FeedbackType = {
+  option?: string
+  message?: string
+  question?: string
+  answer: string
+  pageid?: string
+  citations?: Citation[]
+  type: 'human' | 'bot' | 'error'
+}
+export const logFeedback = async (feedback: FeedbackType) =>
+  fetch(`/chat/log`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(feedback),
+  })
 
 type FeedbackProps = {
   pageid: string
@@ -11,8 +30,17 @@ type FeedbackProps = {
   upHint?: string
   downHint?: string
   options?: string[]
+  onSubmit?: (message: string, option?: string) => Promise<any>
 }
-const Feedback = ({pageid, showForm, labels, upHint, downHint, options}: FeedbackProps) => {
+const Feedback = ({
+  pageid,
+  showForm,
+  labels,
+  upHint,
+  downHint,
+  options,
+  onSubmit,
+}: FeedbackProps) => {
   const [showFeedback, setShowFeedback] = useState(false)
   const [showFeedbackForm, setShowFeedbackForm] = useState(false)
 
@@ -47,6 +75,7 @@ const Feedback = ({pageid, showForm, labels, upHint, downHint, options}: Feedbac
 
       {showFeedbackForm && (
         <FeedbackForm
+          onSubmit={onSubmit}
           onClose={() => {
             setShowFeedback(true)
             setShowFeedbackForm(false)

app/hooks/useChat.ts

@@ -32,6 +32,7 @@ export type UserEntry = {
 
 export type AssistantEntry = {
   role: 'assistant'
+  question?: string
   content: string
   citations?: Citation[]
   citationsMap?: Map<string, Citation>
@@ -82,6 +83,7 @@ export type EntryRole = 'error' | 'stampy' | 'assistant' | 'user' | 'deleted'
 export type HistoryEntry = {
   role: EntryRole
   content: string
+  question?: string
 }
 
 export const formatCitations: (text: string) => string = (text) => {
@@ -181,29 +183,32 @@ export async function* iterateData(res: Response) {
   }
 }
 
-const makeEntry = () =>
+const makeEntry = (question?: string) =>
   ({
     role: 'assistant',
+    question,
     content: '',
     citations: [],
     citationsMap: new Map(),
   }) as AssistantEntry
 
 export const extractAnswer = async (
+  question: string | undefined,
   res: Response,
   setCurrent: (e: AssistantEntry) => void
 ): Promise<SearchResult> => {
   const formatResponse = (result: AssistantEntry, data: Entry) => {
     const content = formatCitations((result?.content || '') + data.content)
     return {
       content,
+      question,
       role: 'assistant',
       citations: result?.citations || [],
       citationsMap: findCitations(content, result?.citations || []),
     } as AssistantEntry
   }
 
-  let result: AssistantEntry = makeEntry()
+  let result: AssistantEntry = makeEntry(question)
   let followups: Followup[] = []
   for await (const data of iterateData(res)) {
     switch (data.state) {
@@ -264,7 +269,8 @@ export const queryLLM = async (
   controller: AbortController,
   settings?: ChatSettings
 ): Promise<SearchResult> => {
-  setCurrent({...makeEntry(), phase: 'started'})
+  const question = history[history.length - 1]?.content
+  setCurrent({...makeEntry(question), phase: 'started'})
   // do SSE on a POST request.
   const res = await fetchLLM(sessionId, history, controller, settings)
 
@@ -275,7 +281,7 @@ export const queryLLM = async (
   }
 
   try {
-    return await extractAnswer(res, setCurrent)
+    return await extractAnswer(question, res, setCurrent)
   } catch (e) {
     if ((e as Error)?.name === 'AbortError') {
       return {result: {role: 'error', content: 'aborted'}}

app/routes/chat.log.ts

@@ -0,0 +1,54 @@
+import type {ActionFunctionArgs} from '@remix-run/cloudflare'
+import type {Citation} from '~/hooks/useChat'
+
+const formatCitation = (citation: Citation) => {
+  const items = (['title', 'date', 'url', 'source', 'text'] as (keyof Citation)[])
+    .map(
+      (key) =>
+        `\t\t<div>\n\t\t\t<span>${key}</span>\n\t\t\t<span>${citation[key]}</span>\n\t\t</div>`
+    )
+    .join('\n')
+  return `\t<div>\n${items}\n\t</div>`
+}
+const formatAnswer = (question?: string, answer?: string, citations?: Citation[]) => {
+  const formattedCitations = citations?.map(formatCitation).join('\n')
+  return `<html>
+              <h1 class="question">${question || '<no question>'}</h1>
+              <article>
+                 <div class="answer">${answer}</div>
+                 <div class="citations">${formattedCitations}</div>
+              </article>
+           </html>`
+}
+
+export const action = async ({request}: ActionFunctionArgs) => {
+  const {question, answer, citations, type, message, option, pageid} = (await request.json()) as any
+
+  try {
+    // Convert the large string content to a Blob
+    const fileBlob = new Blob([formatAnswer(question, answer, citations)], {type: 'text/plain'})
+
+    // Prepare the form data
+    const formData = new FormData()
+    formData.append('file', fileBlob, 'question_and_response.html')
+
+    const info = [
+      ['Type', type],
+      ['Pageid', pageid && `[${pageid}](https://aisafety.info/questions/${pageid}/${question})`],
+      ['Selected option', option],
+      ['Feedback', message],
+    ]
+      .filter(([_, val]) => val && val.trim())
+      .map(([item, val]) => `* ${item} - ${val}`)
+      .join('\n')
+    formData.append('payload_json', JSON.stringify({content: `Chat feedback:\n${info}`}))
+
+    const response = await fetch(`${DISCORD_LOGGING_URL}`, {method: 'POST', body: formData})
+    if (!response.ok) {
+      throw new Error(`Failed to post message: ${response.status} ${response.statusText}`)
+    }
+  } catch (error) {
+    console.error('Failed to post message:', error)
+  }
+  return null
+}

remix.env.d.ts

@@ -10,3 +10,4 @@ declare const CODA_WRITES_TOKEN: string
 declare const NLP_SEARCH_ENDPOINT: string
 declare const ALLOW_ORIGINS: string
 declare const GOOGLE_ANALYTICS_ID: string
+declare const DISCORD_LOGGING_URL: string

wrangler.toml.template

@@ -21,4 +21,5 @@ CODA_INCOMING_TOKEN = "{CODA_INCOMING_TOKEN}"
 CODA_WRITES_TOKEN = "{CODA_WRITES_TOKEN}"
 NLP_SEARCH_ENDPOINT = "https://stampy-nlp-t6p37v2uia-uw.a.run.app/"
 ALLOW_ORIGINS = "https://chat.aisafety.info"
-GOOGLE_ANALYTICS_ID = "{GOOGLE_ANALYTICS_ID}"
+GOOGLE_ANALYTICS_ID = "{GOOGLE_ANALYTICS_ID}"
+DISCORD_LOGGING_URL = "{DISCORD_LOGGING_URL}"