Merge pull request #428 from StampyAI/tags-loading

Filter out internal tags

app/components/Article/Contents.tsx

@@ -1,4 +1,5 @@
 import {useRef, useEffect} from 'react'
+import {questionUrl} from '~/routesMapper'
 import type {Glossary, PageId, GlossaryEntry} from '~/server-utils/stampy'
 
 const footnoteHTML = (el: HTMLDivElement, e: HTMLAnchorElement): string | null => {
@@ -56,27 +57,19 @@ const updateTextNodes = (el: Node, textProcessor: (node: Node) => Node) => {
  *  - use each glossary item only once
  */
 const glossaryInjecter = (pageid: string, glossary: Glossary) => {
-  const unusedGlossaryEntries = Object.values(glossary)
-    .filter((item) => item.pageid != pageid)
-    .map(({term}) => term)
-    .sort((a, b) => b.length - a.length)
-    .map(
-      (term) =>
-        [
-          new RegExp(`(^|[^\\w-])(${term})($|[^\\w-])`, 'i'),
-          '$1<span class="glossary-entry">$2</span>$3',
-        ] as const
-    )
-
-  return (html: string) => {
-    return unusedGlossaryEntries.reduce((html, [match, replacement], index) => {
-      if (html.match(match)) {
-        unusedGlossaryEntries.splice(index, 1)
-        return html.replace(match, replacement)
-      }
-      return html
-    }, html)
-  }
+  const seen = new Set()
+  return (html: string) =>
+    Object.values(glossary)
+      .filter((item) => item.pageid != pageid)
+      .sort((a, b) => b.alias.length - a.alias.length)
+      .reduce((html, {term, alias}) => {
+        const match = new RegExp(`(^|[^\\w-])(${alias})($|[^\\w-])`, 'i')
+        if (!seen.has(term) && html.search(match) >= 0) {
+          seen.add(term)
+          return html.replace(match, '$1<span class="glossary-entry">$2</span>$3')
+        }
+        return html
+      }, html)
 }
 
 const insertGlossary = (pageid: string, glossary: Glossary) => {
@@ -101,6 +94,7 @@ const insertGlossary = (pageid: string, glossary: Glossary) => {
      */
     const glossaryEntry = (e: Element) => {
       const entry = e.textContent && glossary[e?.textContent.toLowerCase().trim()]
+      console.log(e.textContent, entry)
       if (
         // If the contents of this item aren't simply a glossary item word, then
         // something has gone wrong and the glossary-entry should be removed
@@ -124,7 +118,7 @@ const insertGlossary = (pageid: string, glossary: Glossary) => {
       if (!entry) return undefined
       const link =
         entry.pageid &&
-        `<a href="/${entry.pageid}" class="button secondary">View full definition</a>`
+        `<a href="${questionUrl(entry)}" class="button secondary">View full definition</a>`
       const image = entry.image && `<img src="${entry.image}"/>`
       addPopup(
         e as HTMLSpanElement,

app/routes/questions.$questionId.$.tsx

@@ -8,7 +8,7 @@ import {ArticlesNav} from '~/components/ArticlesNav/Menu'
 import {fetchGlossary} from '~/routes/questions.glossary'
 import {loadQuestionDetail, loadTags} from '~/server-utils/stampy'
 import useToC from '~/hooks/useToC'
-import type {Question, Glossary} from '~/server-utils/stampy'
+import type {Question, Glossary, Tag} from '~/server-utils/stampy'
 import {reloadInBackgroundIfNeeded} from '~/server-utils/kv-cache'
 
 export const LINK_WITHOUT_DETAILS_CLS = 'link-without-details'
@@ -59,11 +59,22 @@ const dummyQuestion = (title: string | undefined) =>
     tags: [],
   }) as any as Question
 
+const updateTags = (question: Question, tags: Tag[]) => {
+  const mappedTags = tags.reduce((acc, t) => ({...acc, [t.name]: t}), {})
+  return {
+    ...question,
+    tags: question.tags
+      ?.map((name) => mappedTags[name as keyof typeof mappedTags])
+      .filter((t?: Tag) => t && !t?.internal)
+      .map(({name}) => name),
+  }
+}
+
 export default function RenderArticle() {
   const [glossary, setGlossary] = useState<Glossary>({} as Glossary)
   const params = useParams()
   const pageid = params.questionId ?? '😱'
-  const {data} = useLoaderData<typeof loader>()
+  const {data, tags} = useLoaderData<typeof loader>()
   const {findSection, getArticle, getPath} = useToC()
   const section = findSection(pageid)
 
@@ -83,16 +94,21 @@ export default function RenderArticle() {
           key={pageid}
           fallback={<Article question={dummyQuestion(getArticle(pageid)?.title)} />}
         >
-          <Await resolve={data}>
-            {(resolvedValue) => {
+          <Await resolve={Promise.all([data, tags])}>
+            {([resolvedValue, tags]) => {
               if (resolvedValue instanceof Response) {
                 return <Error error={resolvedValue} />
               } else if (!(resolvedValue as any)?.pageid) {
                 return (
                   <Error error={{statusText: 'Could not fetch question', status: 'emptyArticle'}} />
                 )
               } else {
-                return <Article question={resolvedValue as Question} glossary={glossary} />
+                return (
+                  <Article
+                    question={updateTags(resolvedValue as Question, tags as Tag[])}
+                    glossary={glossary}
+                  />
+                )
               }
             }}
           </Await>

app/routes/questions.toc.ts

@@ -16,6 +16,7 @@ export type TOCItem = {
   hasText: boolean
   children?: TOCItem[]
   category?: Category
+  order: number
 }
 type LoaderResp = {
   data: TOCItem[]
@@ -29,16 +30,19 @@ const getCategory = (tags: string[]): Category => {
   return undefined
 }
 
+const byOrder = (a: TOCItem, b: TOCItem) => a.order - b.order
 const formatQuestion =
   (level: number) =>
-  ({title, pageid, subtitle, icon, children, text, tags}: Question): TOCItem => ({
+  ({title, pageid, subtitle, icon, children, text, tags, order}: Question): TOCItem => ({
     title,
     subtitle: subtitle ? subtitle : undefined,
     pageid,
     icon: icon ? icon : undefined,
     hasText: !!text,
-    children: level < MAX_LEVELS ? children?.map(formatQuestion(level + 1)) : undefined,
+    children:
+      level < MAX_LEVELS ? children?.map(formatQuestion(level + 1)).sort(byOrder) : undefined,
     category: getCategory(tags),
+    order: order || 0,
   })
 
 const getToc = async (request: any) => {

app/routesMapper.ts

@@ -1,5 +1,5 @@
 export const questionUrl = ({pageid, title}: {pageid: string; title?: string}) =>
-  `/questions/${pageid}/${title || ''}`
+  `/questions/${pageid}/${title?.replaceAll(' ', '-') || ''}`
 
 export const tagUrl = ({tagId, name}: {tagId?: number | string; name: string}) =>
   tagId ? `/tags/${tagId}/${name}` : `/tags/${name}`

app/server-utils/stampy.ts

@@ -47,6 +47,7 @@ export type Banner = {
 }
 export type GlossaryEntry = {
   term: string
+  alias: string
   pageid: PageId
   contents: string
   image: string
@@ -79,6 +80,7 @@ export type Question = {
   icon?: string
   parents?: string[]
   children?: Question[]
+  order?: number
 }
 export type PageId = Question['pageid']
 export type NewQuestion = {
@@ -132,6 +134,7 @@ export type AnswersRow = CodaRowCommon & {
     Subtitle?: string
     Icon?: string
     Parents?: Entity[]
+    Order?: number
   }
 }
 type TagsRow = CodaRowCommon & {
@@ -291,6 +294,7 @@ const convertToQuestion = ({name, values, updatedAt} = {} as AnswersRow): Questi
   icon: extractText(values.Icon),
   parents: !values.Parents ? [] : values.Parents?.map(({name}) => name),
   updatedAt: updatedAt || values['Doc Last Edited'],
+  order: values.Order || 0,
 })
 
 export const loadQuestionDetail = withCache('questionDetail', async (question: string) => {
@@ -328,13 +332,14 @@ export const loadGlossary = withCache('loadGlossary', async () => {
         const phrases = [values.phrase, ...values.aliases.split('\n')]
         const item = {
           pageid,
+          term: extractText(values.phrase),
           image: values.image?.url,
           contents: renderText(pageid, extractText(values.definition)),
         }
         return phrases
           .map((i) => extractText(i))
           .filter(Boolean)
-          .map((phrase) => [phrase.toLowerCase(), {term: phrase, ...item}])
+          .map((phrase) => [phrase.toLowerCase(), {alias: phrase, ...item}])
       })
       .flat()
   )

stories/ArticlesNav.stories.ts

@@ -18,85 +18,101 @@ const article = {
   pageid: '9OGZ',
   icon: '/assets/coded-banner.svg',
   hasText: true,
+  order: 0,
   children: [
     {
       title: 'What would an AGI be able to do?',
       pageid: 'NH51',
       hasText: false,
+      order: 0,
     },
     {
       title: 'Types of AI',
       pageid: 'NH50',
       hasText: false,
+      order: 0,
       children: [
         {
           title: 'What are the differences between AGI, transformative AI, and superintelligence?',
           pageid: '5864',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is intelligence?',
           pageid: '6315',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is artificial general intelligence (AGI)?',
           pageid: '2374',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is "superintelligence"?',
           pageid: '6207',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is artificial intelligence (AI)?',
           pageid: '8G1H',
           hasText: true,
+          order: 0,
         },
       ],
     },
     {
       title: 'Introduction to ML',
       pageid: 'NH50',
       hasText: false,
+      order: 0,
       children: [
         {
           title: 'What are large language models?',
           pageid: '8161',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is compute?',
           pageid: '9358',
           hasText: true,
+          order: 0,
         },
       ],
     },
     {
       title: 'Introduction to AI Safety',
       pageid: 'NH53',
       hasText: false,
+      order: 0,
       children: [
         {
           title: 'Why would an AI do bad things?',
           pageid: '2400',
           hasText: true,
+          order: 0,
         },
         {
           title: 'How likely is extinction from superintelligent AI?',
           pageid: '7715',
           hasText: true,
+          order: 0,
         },
         {
           title: 'What is AI safety?',
           pageid: '8486',
           hasText: true,
+          order: 0,
         },
         {
           title: 'Why is safety important for smarter-than-human AI?',
           pageid: '6297',
           hasText: true,
+          order: 0,
         },
       ],
     },

stories/Grid.stories.tsx

@@ -8,55 +8,63 @@ const toc = [
     icon: 'https://cataas.com/cat/says/section1',
     pageid: 'https://google.com',
     hasText: true,
+    order: 0,
   },
   {
     title: 'Governance',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section2',
     pageid: 'https://google.com',
     hasText: true,
+    order: 0,
   },
   {
     title: 'Existential risk concepts',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section3',
     pageid: 'https://google.com',
     hasText: true,
+    order: 0,
   },
   {
     title: 'Predictions on advanced AI',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section4',
     pageid: 'https://google.com',
     hasText: true,
+    order: 0,
   },
   {
     title: 'Prominent research organizations',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section5',
     pageid: 'https://google.com',
     hasText: true,
+    order: 0,
   },
   {
     title: '6th item',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section6',
     pageid: 'https://google.com',
     hasText: true,
+    order: 10,
   },
   {
     title: '7th item',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section7',
     pageid: 'https://google.com',
     hasText: true,
+    order: 20,
   },
   {
     title: '8th item',
     subtitle: 'Lorem ipsum dolor sit amet consectetur',
     icon: 'https://cataas.com/cat/says/section8',
     pageid: 'https://google.com',
     hasText: true,
+    order: 30,
   },
 ]