Merge pull request #622 from StampyAI/chat-fixes

Chat fixes

.github/workflows/deploy-dev.yml

@@ -25,6 +25,7 @@ jobs:
             | sed s/{CODA_INCOMING_TOKEN}/${{ secrets.CODA_INCOMING_TOKEN }}/ \
             | sed s/{CODA_WRITES_TOKEN}/${{ secrets.CODA_WRITES_TOKEN }}/ \
             | sed s/{GOOGLE_ANALYTICS_ID}/${{ secrets.GOOGLE_ANALYTICS_ID }}/ \
+            | sed s/{DISCORD_LOGGING_URL}/${{ secrets.DISCORD_LOGGING_URL }}/ \
             > wrangler.toml
           npm ci
           npm run deploy

.github/workflows/deploy.yml

@@ -24,6 +24,7 @@ jobs:
             | sed s/{CODA_INCOMING_TOKEN}/${{ secrets.CODA_INCOMING_TOKEN }}/ \
             | sed s/{CODA_WRITES_TOKEN}/${{ secrets.CODA_WRITES_TOKEN }}/ \
             | sed s/{GOOGLE_ANALYTICS_ID}/${{ secrets.GOOGLE_ANALYTICS_ID }}/ \
+            | sed s/{DISCORD_LOGGING_URL}/${{ secrets.DISCORD_LOGGING_URL }}/ \
             > wrangler.toml
           npm ci
           npm run deploy

app/components/Chatbot/ChatEntry.tsx

@@ -3,7 +3,7 @@ import {Link} from '@remix-run/react'
 import MarkdownIt from 'markdown-it'
 import QuestionMarkIcon from '~/components/icons-generated/QuestionMark'
 import Contents from '~/components/Article/Contents'
-import Feedback from '~/components/Feedback'
+import Feedback, {logFeedback} from '~/components/Feedback'
 import useGlossary from '~/hooks/useGlossary'
 import './chat_entry.css'
 import type {Entry, AssistantEntry, StampyEntry, Citation, ErrorMessage} from '~/hooks/useChat'
@@ -136,7 +136,7 @@ const Reference = (citation: Citation) => {
   )
 }
 
-const ChatbotReply = ({phase, content, citationsMap}: AssistantEntry) => {
+const ChatbotReply = ({question, phase, content, citationsMap}: AssistantEntry) => {
   const citations = [] as Citation[]
   citationsMap?.forEach((v) => {
     citations.push(v)
@@ -193,6 +193,9 @@ const ChatbotReply = ({phase, content, citationsMap}: AssistantEntry) => {
           pageid="chatbot"
           upHint="This response was helpful"
           downHint="This response was unhelpful"
+          onSubmit={async (message: string, option?: string) =>
+            logFeedback({message, option, type: 'bot', question, answer: content, citations})
+          }
           options={[
             'Making things up',
             'Wrong subject',
@@ -227,6 +230,9 @@ const StampyArticle = ({pageid, content, title}: StampyEntry) => {
           pageid={pageid}
           upHint="This response was helpful"
           downHint="This response was unhelpful"
+          onSubmit={async (message: string, option?: string) =>
+            logFeedback({message, option, type: 'human', question: title, answer: content, pageid})
+          }
           options={[
             'Making things up',
             'Wrong subject',

app/components/Chatbot/index.tsx

@@ -13,46 +13,35 @@ import Input from '~/components/Input'
 
 // to be replaced with actual pool questions
 const poolQuestions = [
+  {title: 'Do people seriously worry about existential risk from AI?', pageid: '6953'},
+  {title: 'Is AI safety about systems becoming malevolent or conscious?', pageid: '6194'},
+  {title: 'When do experts think human-level AI will be created?', pageid: '5633'},
+  {title: 'Why is AI alignment a hard problem?', pageid: '8163'},
   {
-    title: 'What is AI Safety? - from pool',
-    pageid: '8486',
+    title: 'Why can’t we just “put the AI in a box” so that it can’t influence the outside world?',
+    pageid: '6176',
   },
   {
-    title: 'How would the AI even get out in the world? -- from pool',
-    pageid: '7638',
+    title: 'What are the differences between AGI, transformative AI, and superintelligence?',
+    pageid: '5864',
   },
+  {title: 'What are large language models?', pageid: '5864'},
+  {title: "Why can't we just turn the AI off if it starts to misbehave?", pageid: '3119'},
+  {title: 'What is instrumental convergence?', pageid: '897I'},
+  {title: "What is Goodhart's law?", pageid: '8185'},
+  {title: 'What is the orthogonality thesis?', pageid: '6568'},
+  {title: 'How powerful would a superintelligence become?', pageid: '7755'},
+  {title: 'Will AI be able to think faster than humans?', pageid: '8E41'},
+  {title: "Isn't the real concern misuse?", pageid: '9B85'},
+  {title: 'Are AIs conscious?', pageid: '8V5J'},
   {
-    title: 'What is the AI alignment problem? -- from pool',
-    pageid: '8EL9',
-  },
-  {
-    title: 'What are existential risks (x-risks)? -- from pool',
-    pageid: '89LL',
-  },
-  {
-    title: "Isn't the real concern misuse? -- from pool",
-    pageid: '9B85',
-  },
-  {
-    title: "Aren't there easy solutions to AI alignment? -- from pool",
-    pageid: '6172',
-  },
-  {
-    title: 'Will we ever build superintelligence? -- from pool',
-    pageid: '7565',
-  },
-  {
-    title: 'Will the first AGI be an LLM? -- from pool',
-    pageid: '85E2',
-  },
-  {
-    title: 'Why not just raise AI like kids? -- from pool',
-    pageid: '93R9',
-  },
-  {
-    title: 'Why is AI alignment a hard problem? -- from pool',
-    pageid: '8163',
+    title:
+      'What are the differences between a singularity, an intelligence explosion, and a hard takeoff?',
+    pageid: '8IHO',
   },
+  {title: 'What is an intelligence explosion?', pageid: '6306'},
+  {title: 'How might AGI kill people?', pageid: '5943'},
+  {title: 'What is a "warning shot"?', pageid: '7748'},
 ]
 
 const MIN_SIMILARITY = 0.85
@@ -105,7 +94,7 @@ const QuestionInput = ({initial, onChange, onAsk}: QuestionInputProps) => {
   )
 }
 
-export const WidgetStampy = () => {
+export const WidgetStampy = ({className}: {className?: string}) => {
   const [question, setQuestion] = useState('')
   const navigate = useNavigate()
   const questions = [
@@ -116,7 +105,7 @@ export const WidgetStampy = () => {
 
   const stampyUrl = (question: string) => `/chat/?question=${question.trim()}`
   return (
-    <div className="centered fcol-9 padding-bottom-128">
+    <div className={`centered fcol-9 padding-bottom-128 ${className || ''}`}>
       <div className="fcol-6 padding-bottom-56">
         <h2 className="teal-500">Questions?</h2>
         <h2>Ask Stampy, our chatbot, any question about AI safety</h2>
@@ -257,6 +246,7 @@ export const Chatbot = ({question, questions, settings}: ChatbotProps) => {
 
     // Add a new history entry, replacing the previous one if it was canceled
     const message = {content: question, role: 'user'} as Entry
+    const answer = {role: 'assistant', question} as AssistantEntry
     setHistory((current) => {
       const last = current[current.length - 1]
       if (
@@ -265,15 +255,11 @@ export const Chatbot = ({question, questions, settings}: ChatbotProps) => {
         (last?.role === 'stampy' && last?.content) ||
         ['error'].includes(last?.role)
       ) {
-        return [...current, message, {role: 'assistant'} as AssistantEntry]
+        return [...current, message, answer]
       } else if (last?.role === 'user' && last?.content === question) {
-        return [...current.slice(0, current.length - 1), {role: 'assistant'} as AssistantEntry]
+        return [...current.slice(0, current.length - 1), answer]
       }
-      return [
-        ...current.slice(0, current.length - 2),
-        message,
-        {role: 'assistant'} as AssistantEntry,
-      ]
+      return [...current.slice(0, current.length - 2), message, answer]
     })
 
     setFollowups(undefined)

app/components/Feedback/Form.tsx

@@ -4,11 +4,13 @@ import useOutsideOnClick from '~/hooks/useOnOutsideClick'
 import './feedback.css'
 
 export type FeedbackFormProps = {
+  onSubmit?: (msg: string, option?: string) => Promise<any>
   onClose?: () => void
   options?: string[]
 }
-const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
+const FeedbackForm = ({onSubmit, onClose, options}: FeedbackFormProps) => {
   const [selected, setSelected] = useState<string>()
+  const [message, setMessage] = useState('')
   const [enabledSubmit, setEnabledSubmit] = useState(!options)
   const [numClicks, setNumClicks] = useState(0)
   const clickCheckerRef = useOutsideOnClick(onClose)
@@ -28,7 +30,8 @@ const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
     setEnabledSubmit(true)
   }
 
-  const handleSubmit = () => {
+  const handleSubmit = async () => {
+    onSubmit && (await onSubmit(message, selected))
     onClose && onClose()
   }
 
@@ -56,6 +59,7 @@ const FeedbackForm = ({onClose, options}: FeedbackFormProps) => {
         name="feedback-text"
         className={['feedback-text bordered', !options ? 'no-options' : ''].join(' ')}
         placeholder="Leave a comment (optional)"
+        onChange={(e) => setMessage(e.target.value)}
       />
       <Button className="primary full-width" action={handleSubmit} disabled={!enabledSubmit}>
         <p>Submit feedback</p>

app/components/Feedback/index.tsx

@@ -3,6 +3,25 @@ import {CompositeButton} from '~/components/Button'
 import {Action, ActionType} from '~/routes/questions.actions'
 import './feedback.css'
 import FeedbackForm from './Form'
+import type {Citation} from '~/hooks/useChat'
+
+type FeedbackType = {
+  option?: string
+  message?: string
+  question?: string
+  answer: string
+  pageid?: string
+  citations?: Citation[]
+  type: 'human' | 'bot' | 'error'
+}
+export const logFeedback = async (feedback: FeedbackType) =>
+  fetch(`/chat/log`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(feedback),
+  })
 
 type FeedbackProps = {
   pageid: string
@@ -11,8 +30,17 @@ type FeedbackProps = {
   upHint?: string
   downHint?: string
   options?: string[]
+  onSubmit?: (message: string, option?: string) => Promise<any>
 }
-const Feedback = ({pageid, showForm, labels, upHint, downHint, options}: FeedbackProps) => {
+const Feedback = ({
+  pageid,
+  showForm,
+  labels,
+  upHint,
+  downHint,
+  options,
+  onSubmit,
+}: FeedbackProps) => {
   const [showFeedback, setShowFeedback] = useState(false)
   const [showFeedbackForm, setShowFeedbackForm] = useState(false)
 
@@ -47,6 +75,7 @@ const Feedback = ({pageid, showForm, labels, upHint, downHint, options}: Feedbac
 
       {showFeedbackForm && (
         <FeedbackForm
+          onSubmit={onSubmit}
           onClose={() => {
             setShowFeedback(true)
             setShowFeedbackForm(false)

app/components/SearchResults/Dropdown.tsx

@@ -27,7 +27,7 @@ export const SearchResults = ({results}: {results: SearchResultsProps[]}) => {
   const noResults = results.length === 0
   if (noResults) {
     return (
-      <div className="container-search-results bordered col-5 container-search-results-mobile">
+      <div className="full-width container-search-results bordered fcol-5 container-search-results-mobile">
         <div className="search-result">No results found</div>
       </div>
     )

app/components/search.tsx

@@ -39,7 +39,7 @@ export default function Search({limitFromUrl}: Props) {
       {isPendingSearch && results.length == 0 && (
         <div className="result-item-box no-questions">Searching for questions...</div>
       )}
-      {searchPhrase && showResults && (
+      {!isPendingSearch && searchPhrase && showResults && (
         <SearchResults
           results={results.map((r) => ({
             title: r.title,

app/hooks/useChat.ts

@@ -32,6 +32,7 @@ export type UserEntry = {
 
 export type AssistantEntry = {
   role: 'assistant'
+  question?: string
   content: string
   citations?: Citation[]
   citationsMap?: Map<string, Citation>
@@ -82,6 +83,7 @@ export type EntryRole = 'error' | 'stampy' | 'assistant' | 'user' | 'deleted'
 export type HistoryEntry = {
   role: EntryRole
   content: string
+  question?: string
 }
 
 export const formatCitations: (text: string) => string = (text) => {
@@ -181,29 +183,32 @@ export async function* iterateData(res: Response) {
   }
 }
 
-const makeEntry = () =>
+const makeEntry = (question?: string) =>
   ({
     role: 'assistant',
+    question,
     content: '',
     citations: [],
     citationsMap: new Map(),
   }) as AssistantEntry
 
 export const extractAnswer = async (
+  question: string | undefined,
   res: Response,
   setCurrent: (e: AssistantEntry) => void
 ): Promise<SearchResult> => {
   const formatResponse = (result: AssistantEntry, data: Entry) => {
     const content = formatCitations((result?.content || '') + data.content)
     return {
       content,
+      question,
       role: 'assistant',
       citations: result?.citations || [],
       citationsMap: findCitations(content, result?.citations || []),
     } as AssistantEntry
   }
 
-  let result: AssistantEntry = makeEntry()
+  let result: AssistantEntry = makeEntry(question)
   let followups: Followup[] = []
   for await (const data of iterateData(res)) {
     switch (data.state) {
@@ -264,7 +269,8 @@ export const queryLLM = async (
   controller: AbortController,
   settings?: ChatSettings
 ): Promise<SearchResult> => {
-  setCurrent({...makeEntry(), phase: 'started'})
+  const question = history[history.length - 1]?.content
+  setCurrent({...makeEntry(question), phase: 'started'})
   // do SSE on a POST request.
   const res = await fetchLLM(sessionId, history, controller, settings)
 
@@ -275,7 +281,7 @@ export const queryLLM = async (
   }
 
   try {
-    return await extractAnswer(res, setCurrent)
+    return await extractAnswer(question, res, setCurrent)
   } catch (e) {
     if ((e as Error)?.name === 'AbortError') {
       return {result: {role: 'error', content: 'aborted'}}

app/routes/_index.tsx

@@ -45,7 +45,7 @@ export default function App() {
         <ContentBoxThird />
 
         <div className="desktop-only padding-bottom-56" />
-        <WidgetStampy />
+        <WidgetStampy className="desktop-only" />
 
         <h3 className="grey large-bold padding-bottom-32">Advanced sections</h3>
         <Grid gridBoxes={advanced} />