Merge pull request #3977 from owid/data-catalog-algolia

🎉 Data Catalog

.bundlewatch.config.json

@@ -6,7 +6,7 @@
         },
         {
             "path": "./dist/assets/owid.css",
-            "maxSize": "430KB"
+            "maxSize": "450KB"
         }
     ],
     "defaultCompression": "none"

adminSiteServer/mockSiteRouter.tsx

@@ -4,7 +4,7 @@ import {
     renderFrontPage,
     renderGdocsPageBySlug,
     renderPageBySlug,
-    renderChartsPage,
+    renderDataCatalogPage,
     renderSearchPage,
     renderDonatePage,
     makeAtomFeed,
@@ -327,10 +327,9 @@ getPlainRouteNonIdempotentWithRWTransaction(
 
 getPlainRouteWithROTransaction(
     mockSiteRouter,
-    "/charts",
+    "/data*",
     async (req, res, trx) => {
-        const explorerAdminServer = new ExplorerAdminServer(GIT_CMS_DIR)
-        res.send(await renderChartsPage(trx, explorerAdminServer))
+        res.send(await renderDataCatalogPage(trx))
     }
 )
 

baker/SiteBaker.tsx

@@ -15,7 +15,7 @@ import {
 import {
     renderFrontPage,
     renderBlogByPageNum,
-    renderChartsPage,
+    renderDataCatalogPage,
     renderSearchPage,
     renderDonatePage,
     makeAtomFeed,
@@ -676,8 +676,8 @@ export class SiteBaker {
         )
 
         await this.stageWrite(
-            `${this.bakedSiteDir}/charts.html`,
-            await renderChartsPage(knex, this.explorerAdminServer)
+            `${this.bakedSiteDir}/data.html`,
+            await renderDataCatalogPage(knex)
         )
         this.progressBar.tick({ name: "✅ baked special pages" })
     }

baker/algolia/configureAlgolia.ts

@@ -56,8 +56,6 @@ export const configureAlgolia = async () => {
         exactOnSingleWordQuery: "none",
         removeStopWords: ["en"],
         snippetEllipsisText: "…",
-        highlightPreTag: "<strong>",
-        highlightPostTag: "</strong>",
         distinct: true,
         advancedSyntax: true,
         advancedSyntaxFeatures: ["exactPhrase"],
@@ -303,6 +301,8 @@ export const configureAlgolia = async () => {
         ["solar", "photovoltaic", "photovoltaics", "pv"],
         ["tb", "tuberculosis"],
         ["ntd", "neglected tropical diseases", "neglected tropical disease"],
+        ["people", "population"],
+        ["production", "generation"],
     ]
 
     const algoliaSynonyms = synonyms.map((s) => {

baker/algolia/indexChartsToAlgolia.ts

@@ -11,6 +11,7 @@ import {
     countries,
     orderBy,
     removeTrailingParenthetical,
+    uniq,
 } from "@ourworldindata/utils"
 import { MarkdownTextWrap } from "@ourworldindata/components"
 import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js"
@@ -162,6 +163,8 @@ const getChartsRecords = async (
 
     const pageviews = await getAnalyticsPageviewsByUrlObj(knex)
 
+    const parentTagsByChildName = await db.getParentTagsByChildName(knex)
+
     const records: ChartRecord[] = []
     for (const c of parsedRows) {
         // Our search currently cannot render explorers, so don't index them because
@@ -182,6 +185,11 @@ const getChartsRecords = async (
                   fontSize: 10, // doesn't matter, but is a mandatory field
               }).plaintext
 
+        const parentTags = c.tags.flatMap(
+            // a chart can be tagged with a tag that isn't in the tag graph
+            (tag) => parentTagsByChildName[tag] || []
+        )
+
         const record = {
             objectID: c.id.toString(),
             chartId: c.id,
@@ -193,7 +201,7 @@ const getChartsRecords = async (
             numDimensions: parseInt(c.numDimensions),
             publishedAt: c.publishedAt,
             updatedAt: c.updatedAt,
-            tags: c.tags as any as string[],
+            tags: uniq([...c.tags, ...parentTags]),
             keyChartForTags: c.keyChartForTags as string[],
             titleLength: c.title.length,
             // Number of references to this chart in all our posts and pages

baker/siteRenderers.tsx

@@ -1,6 +1,6 @@
 import { LongFormPage, PageOverrides } from "../site/LongFormPage.js"
 import { BlogIndexPage } from "../site/BlogIndexPage.js"
-import { ChartsIndexPage, ChartIndexItem } from "../site/ChartsIndexPage.js"
+import { DataCatalogPage } from "../site/DataCatalog/DataCatalogPage.js"
 import { DynamicCollectionPage } from "../site/collections/DynamicCollectionPage.js"
 import { StaticCollectionPage } from "../site/collections/StaticCollectionPage.js"
 import { SearchPage } from "../site/search/SearchPage.js"
@@ -44,6 +44,7 @@ import {
     OwidGdoc,
     OwidGdocDataInsightInterface,
     mergeGrapherConfigs,
+    createTagGraph,
 } from "@ourworldindata/utils"
 import { extractFormattingOptions } from "../serverUtils/wordpressUtils.js"
 import {
@@ -60,6 +61,7 @@ import {
     KnexReadWriteTransaction,
     KnexReadonlyTransaction,
     getHomepageId,
+    getFlatTagGraph,
 } from "../db/db.js"
 import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js"
 import { ProminentLink } from "../site/blocks/ProminentLink.js"
@@ -100,60 +102,11 @@ import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js"
 export const renderToHtmlPage = (element: any) =>
     `<!doctype html>${ReactDOMServer.renderToStaticMarkup(element)}`
 
-export const renderChartsPage = async (
-    knex: KnexReadonlyTransaction,
-    explorerAdminServer: ExplorerAdminServer
-) => {
-    const explorers = await explorerAdminServer.getAllPublishedExplorers()
-
-    const chartItems = await knexRaw<ChartIndexItem>(
-        knex,
-        `-- sql
-        SELECT
-            c.id,
-            cc.slug,
-            cc.full->>"$.title" AS title,
-            cc.full->>"$.variantName" AS variantName
-        FROM charts c
-        JOIN chart_configs cc ON c.configId=cc.id
-        WHERE
-            c.isIndexable IS TRUE
-            AND c.publishedAt IS NOT NULL
-            AND cc.full->>"$.isPublished" = "true"
-    `
-    )
-
-    const chartTags = await knexRaw<{
-        chartId: number
-        tagId: number
-        tagName: string
-        tagParentId: number
-    }>(
-        knex,
-        `-- sql
-        SELECT ct.chartId, ct.tagId, t.name as tagName, t.parentId as tagParentId FROM chart_tags ct
-        JOIN charts c ON c.id=ct.chartId
-        JOIN tags t ON t.id=ct.tagId
-    `
-    )
-
-    for (const c of chartItems) {
-        c.tags = []
-    }
-
-    const chartsById = lodash.keyBy(chartItems, (c) => c.id)
-
-    for (const ct of chartTags) {
-        const c = chartsById[ct.chartId]
-        if (c) c.tags.push({ id: ct.tagId, name: ct.tagName })
-    }
-
+export const renderDataCatalogPage = async (knex: KnexReadonlyTransaction) => {
+    const { __rootId, ...flatTagGraph } = await getFlatTagGraph(knex)
+    const rootTagGraph = createTagGraph(flatTagGraph, __rootId)
     return renderToHtmlPage(
-        <ChartsIndexPage
-            explorers={explorers}
-            chartItems={chartItems}
-            baseUrl={BAKED_BASE_URL}
-        />
+        <DataCatalogPage baseUrl={BAKED_BASE_URL} tagGraph={rootTagGraph} />
     )
 }
 

db/db.ts

@@ -7,7 +7,7 @@ import {
     GRAPHER_DB_PORT,
 } from "../settings/serverSettings.js"
 import { registerExitHandler } from "./cleanup.js"
-import { keyBy } from "@ourworldindata/utils"
+import { createTagGraph, keyBy } from "@ourworldindata/utils"
 import {
     DbChartTagJoin,
     ImageMetadata,
@@ -26,8 +26,10 @@ import {
     DbPlainPostGdocLink,
     OwidGdocLinkType,
     OwidGdoc,
+    DbPlainTag,
+    TagGraphNode,
 } from "@ourworldindata/types"
-import { groupBy } from "lodash"
+import { groupBy, uniq } from "lodash"
 import { gdocFromJSON } from "./model/Gdoc/GdocFactory.js"
 
 // Return the first match from a mysql query
@@ -527,6 +529,45 @@ export async function getFlatTagGraph(knex: KnexReadonlyTransaction): Promise<
     return { ...tagGraphByParentId, __rootId: tagGraphRootIdResult.id }
 }
 
+// DFS through the tag graph and create a map of parent tags for each child tag
+// e.g. { "Child": [ "Parent", "Grandparent" ], "Parent": [ "Grandparent" ] }
+// parent tags are listed in no particular order
+export async function getParentTagsByChildName(
+    trx: KnexReadonlyTransaction
+): Promise<Record<DbPlainTag["name"], DbPlainTag["name"][]>> {
+    const { __rootId, ...flatTagGraph } = await getFlatTagGraph(trx)
+    const tagGraph = createTagGraph(flatTagGraph, __rootId)
+
+    const tagsById = await trx("tags")
+        .select("id", "name")
+        .then((tags) => keyBy(tags, "id"))
+
+    const parentTagsByChildName: Record<
+        DbPlainTag["name"],
+        DbPlainTag["name"][]
+    > = {}
+
+    function trackParents(node: TagGraphNode): void {
+        for (const child of node.children) {
+            trackParents(child)
+        }
+
+        const preexistingParents = parentTagsByChildName[node.name] ?? []
+        // node.path is an array of tag ids from the root to the current node
+        // slice to remove the root node and the current node, then map them into tag names
+        const newParents = node.path.slice(1, -1).map((id) => tagsById[id].name)
+
+        parentTagsByChildName[node.name] = uniq([
+            ...preexistingParents,
+            ...newParents,
+        ])
+    }
+
+    trackParents(tagGraph)
+
+    return parentTagsByChildName
+}
+
 export async function updateTagGraph(
     knex: KnexReadWriteTransaction,
     tagGraph: FlatTagGraph

db/migration/1725917619555-RedirectChartsRouteToDataRoute.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class RedirectChartsRouteToDataRoute1725917619555
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            DELETE FROM redirects WHERE source = "/data"
+        `)
+        await queryRunner.query(`-- sql
+            INSERT INTO redirects (source, target) VALUES ("/charts", "/data")
+        `)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            INSERT INTO redirects (source, target) VALUES ("/data", "#entries")
+        `)
+        await queryRunner.query(`-- sql
+            DELETE FROM redirects WHERE source = "/charts"
+        `)
+    }
+}

devTools/docker/sync-s3-images.sh

@@ -28,4 +28,13 @@ else
     echo "Access Denied to owid-image-upload, fetching from owid-image-upload-staging..."
 fi
 
+echo "Debug: [$GRAPHER_BRANCH]"
+echo "Debug: [$PROD_BUCKET]"
+echo "Debug: [$IMAGE_HOSTING_R2_BUCKET_PATH]"
+
+# Strip CR from the end of the bucket path
+IMAGE_HOSTING_R2_BUCKET_PATH=$(echo $IMAGE_HOSTING_R2_BUCKET_PATH | tr -d '\r')
+
+echo "Debug after tr: [$IMAGE_HOSTING_R2_BUCKET_PATH]"
+
 rclone sync owid-r2:$PROD_BUCKET/production/ owid-r2:$IMAGE_HOSTING_R2_BUCKET_PATH/ --verbose --transfers=32 --checkers=32 --fast-list

package.json

@@ -134,7 +134,7 @@
         "react-flip-toolkit": "^7.0.9",
         "react-hook-form": "^7.51.3",
         "react-horizontal-scrolling-menu": "^4.0.3",
-        "react-instantsearch": "^7.11.3",
+        "react-instantsearch": "^7.12.2",
         "react-intersection-observer": "^9.10.1",
         "react-move": "^6.5.0",
         "react-recaptcha": "^2.3.10",

packages/@ourworldindata/components/src/LabeledSwitch/LabeledSwitch.scss

@@ -92,4 +92,16 @@ $lato: $sans-serif-font-stack;
     &:hover input:checked + .outer .inner {
         background: darken($active-switch, 13%);
     }
+
+    &.labeled-switch--is-disabled {
+        opacity: 0.5;
+        label {
+            cursor: default;
+        }
+        &:hover {
+            .inner {
+                background: $gray-70;
+            }
+        }
+    }
 }

packages/@ourworldindata/components/src/LabeledSwitch/LabeledSwitch.tsx

@@ -2,6 +2,7 @@ import React from "react"
 import { observer } from "mobx-react"
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js"
 import { faInfoCircle } from "@fortawesome/free-solid-svg-icons"
+import cx from "classnames"
 import { Tippy } from "@ourworldindata/utils"
 
 @observer
@@ -11,17 +12,36 @@ export class LabeledSwitch extends React.Component<{
     tooltip?: string
     tracking?: string
     onToggle: () => any
+    className?: string
+    disabled?: boolean
 }> {
     render(): React.ReactElement {
-        const { label, value, tooltip, tracking } = this.props
+        const {
+            className,
+            label,
+            value,
+            tooltip,
+            tracking,
+            disabled,
+            onToggle,
+        } = this.props
 
         return (
-            <div className="labeled-switch">
+            <div
+                className={cx(
+                    {
+                        "labeled-switch": true,
+                        "labeled-switch--is-disabled": disabled,
+                    },
+                    className
+                )}
+            >
                 <label>
                     <input
                         type="checkbox"
                         checked={value}
-                        onChange={this.props.onToggle}
+                        disabled={disabled}
+                        onChange={onToggle}
                         data-track-note={tracking}
                     />
                     <div data-track-note="" className="outer">