mdd: fetch variable-level config by uuid (#3922)

This is currently quite painful to test -- sorry!

During the bake, a `variableIdToGrapherConfigMap` is written into the HTML.
It maps primary variable IDs to their respective `grapherConfigIdAdmin ?? grapherConfigIdETL ?? null`.

If such a UUID is present, then the config is fetched at runtime.
It is currently a bit hard to make grapher wait for this grapher config to arrive - it displays an empty screen during the loading, rather than a loading indicator of some sort.
We can tweak this later on, but we don't want grapher to render _before_ the config override has been fetched, otherwise we're flash-rendering twice with different FAUST.

baker/MultiDimBaker.tsx

@@ -23,6 +23,7 @@ import React from "react"
 import { BAKED_BASE_URL } from "../settings/clientSettings.js"
 import { getTagToSlugMap } from "./GrapherBakingUtils.js"
 import {
+    getGrapherConfigIdsForVariables,
     getVariableIdsByCatalogPath,
     getVariableMetadata,
 } from "../db/model/Variable.js"
@@ -38,6 +39,20 @@ import {
     getMultiDimDataPageBySlug,
 } from "../db/model/MultiDimDataPage.js"
 
+const getGrapherConfigIdsByVariableIds = async (
+    knex: db.KnexReadonlyTransaction,
+    variableIds: number[]
+): Promise<Record<number, string | null>> => {
+    const rows = await getGrapherConfigIdsForVariables(knex, variableIds)
+
+    return Object.fromEntries(
+        rows.map((row) => [
+            row.id,
+            row.grapherConfigIdAdmin ?? row.grapherConfigIdETL,
+        ])
+    )
+}
+
 const resolveMultiDimDataPageCatalogPathsToIndicatorIds = async (
     knex: db.KnexReadonlyTransaction,
     rawConfig: MultiDimDataPageConfigRaw
@@ -206,6 +221,13 @@ export const renderMultiDimDataPageFromConfig = async (
         await resolveMultiDimDataPageCatalogPathsToIndicatorIds(knex, rawConfig)
     const config = MultiDimDataPageConfig.fromObject(preProcessedConfig)
 
+    // GET VARIABLE GRAPHER CONFIG UUIDS
+    const relevantVariableIds = getRelevantVariableIds(preProcessedConfig)
+    const variableIdToGrapherConfigMap = await getGrapherConfigIdsByVariableIds(
+        knex,
+        Array.from(relevantVariableIds)
+    )
+
     // FAQs
     const variableMetaDict =
         await getRelevantVariableMetadata(preProcessedConfig)
@@ -224,6 +246,7 @@ export const renderMultiDimDataPageFromConfig = async (
     const props = {
         configObj: config.config,
         tagToSlugMap: minimalTagToSlugMap,
+        variableIdToGrapherConfigMap,
         faqEntries,
         primaryTopic,
     }

db/model/Variable.ts

@@ -52,6 +52,25 @@ interface VariableWithGrapherConfigs {
     etl?: ChartConfigPair
 }
 
+type VariableWithGrapherConfigIds = Pick<
+    DbRawVariable,
+    "id" | "grapherConfigIdAdmin" | "grapherConfigIdETL"
+>
+export async function getGrapherConfigIdsForVariables(
+    knex: db.KnexReadonlyTransaction,
+    variableIds: number[]
+): Promise<VariableWithGrapherConfigIds[]> {
+    const rows: VariableWithGrapherConfigIds[] = await knex(VariablesTableName)
+        .select([
+            "id",
+            "grapherConfigIdAdmin",
+            "grapherConfigIdETL",
+        ] as (keyof DbRawVariable)[])
+        .whereIn("id", variableIds)
+
+    return rows
+}
+
 export async function getGrapherConfigsForVariable(
     knex: db.KnexReadonlyTransaction,
     variableId: number

packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts

@@ -28,7 +28,7 @@ export interface Dimension {
     name: string
     group?: string
     description?: string
-    multi_select?: boolean
+    // multi_select?: boolean
     choices: Choice[]
 }
 
@@ -83,6 +83,7 @@ export interface MultiDimDataPageProps {
     tagToSlugMap?: Record<string, string>
     faqEntries?: FaqEntryKeyedByGdocIdAndFragmentId
     primaryTopic?: PrimaryTopic | undefined
+    variableIdToGrapherConfigMap?: Record<number, string | null>
 
     initialQueryStr?: string
     canonicalUrl?: string

site/multiDim/MultiDimDataPageContent.tsx

@@ -22,6 +22,7 @@ import {
     MultiDimDataPageConfig,
     extractMultiDimChoicesFromQueryStr,
     multiDimStateToQueryStr,
+    omit,
 } from "@ourworldindata/utils"
 import cx from "classnames"
 import { DebugProvider } from "../gdocs/DebugContext.js"
@@ -101,6 +102,13 @@ const cachedGetVariableMetadata = memoize(
         )
 )
 
+const cachedGetGrapherConfigByUuid = memoize(
+    (grapherConfigUuid: string): Promise<GrapherInterface> =>
+        fetch(`/grapher/by-uuid/${grapherConfigUuid}.config.json`).then(
+            (resp) => resp.json()
+        )
+)
+
 const useTitleFragments = (config: MultiDimDataPageConfig) => {
     const title = config.config.title
     return useMemo(
@@ -127,34 +135,70 @@ const useView = (
 }
 
 const useVarDatapageData = (
-    currentView: View<IndicatorsAfterPreProcessing> | undefined
+    currentView: View<IndicatorsAfterPreProcessing> | undefined,
+    variableIdToGrapherConfigMap: Record<number, string | null> | undefined
 ) => {
     const [varDatapageData, setVarDatapageData] =
         useState<DataPageDataV2 | null>(null)
+    const [grapherConfig, setGrapherConfig] = useState<GrapherInterface | null>(
+        null
+    )
+    const [grapherConfigIsReady, setGrapherConfigIsReady] = useState(false)
 
     useEffect(() => {
+        setGrapherConfigIsReady(false)
+        setGrapherConfig(null)
         setVarDatapageData(null)
         const yIndicatorOrIndicators = currentView?.indicators?.["y"]
         const variableId = Array.isArray(yIndicatorOrIndicators)
             ? yIndicatorOrIndicators[0]
             : yIndicatorOrIndicators
         if (!variableId) return
-        const variableMetadata = cachedGetVariableMetadata(variableId)
 
-        variableMetadata
-            .then((json) => getDatapageDataV2(json, currentView?.config))
-            .then(setVarDatapageData)
+        const datapageDataPromise = cachedGetVariableMetadata(variableId).then(
+            (json) => getDatapageDataV2(json, currentView?.config)
+        )
+        const grapherConfigUuid = variableIdToGrapherConfigMap?.[variableId]
+        const grapherConfigPromise = grapherConfigUuid
+            ? cachedGetGrapherConfigByUuid(grapherConfigUuid)
+            : null
+
+        Promise.allSettled([datapageDataPromise, grapherConfigPromise])
+            .then(([datapageData, grapherConfig]) => {
+                if (datapageData.status === "rejected")
+                    throw new Error(
+                        `Fetching variable by uuid failed: ${grapherConfigUuid}`,
+                        { cause: datapageData.reason }
+                    )
+
+                setVarDatapageData(datapageData.value)
+                setGrapherConfig(
+                    grapherConfig.status === "fulfilled"
+                        ? grapherConfig.value
+                        : null
+                )
+                setGrapherConfigIsReady(true)
+            })
             .catch(console.error)
-    }, [currentView?.indicators, currentView?.config])
+    }, [
+        currentView?.indicators,
+        currentView?.config,
+        variableIdToGrapherConfigMap,
+    ])
 
-    return { varDatapageData }
+    return {
+        varDatapageData,
+        varGrapherConfig: grapherConfig,
+        grapherConfigIsReady,
+    }
 }
 
 export const MultiDimDataPageContent = ({
     // _datapageData,
     configObj,
     // isPreviewing = false,
     faqEntries,
+    variableIdToGrapherConfigMap,
     primaryTopic,
     canonicalUrl = "{URL}", // when we bake pages to their proper url this will be set correctly but on preview pages we leave this undefined
     tagToSlugMap,
@@ -179,7 +223,8 @@ export const MultiDimDataPageContent = ({
     })
 
     const { currentView, dimensionsConfig } = useView(currentSettings, config)
-    const { varDatapageData } = useVarDatapageData(currentView)
+    const { varDatapageData, varGrapherConfig, grapherConfigIsReady } =
+        useVarDatapageData(currentView, variableIdToGrapherConfigMap)
 
     // This is the ACTUAL grapher instance being used, because GrapherFigureView/GrapherWithFallback are doing weird things and are not actually using the grapher instance we pass into it
     // and therefore we can not access the grapher state (e.g. tab, selection) from the grapher instance we pass into it
@@ -207,15 +252,27 @@ export const MultiDimDataPageContent = ({
     }, [queryStr])
 
     const grapherConfigComputed = useMemo(() => {
+        const baseConfig: GrapherProgrammaticInterface = {
+            isEmbeddedInAnOwidPage: true,
+            selectedEntityNames: config.config.defaultSelection ?? [],
+            bounds,
+        }
+
+        if (!grapherConfigIsReady) return baseConfig
         return {
+            ...varGrapherConfig,
             ...currentView?.config,
             dimensions: dimensionsConfig,
-            isEmbeddedInADataPage: true,
-            selectedEntityNames: config.config.defaultSelection ?? [],
-
-            bounds,
+            ...baseConfig,
         } as GrapherProgrammaticInterface
-    }, [currentView, dimensionsConfig, bounds, config])
+    }, [
+        varGrapherConfig,
+        grapherConfigIsReady,
+        currentView,
+        dimensionsConfig,
+        bounds,
+        config,
+    ])
 
     const hasTopicTags = !!config.config.topicTags?.length
 
@@ -293,7 +350,9 @@ export const MultiDimDataPageContent = ({
                     >
                         <figure data-grapher-src ref={grapherFigureRef}>
                             <Grapher
-                                key={JSON.stringify(currentView)}
+                                key={JSON.stringify(
+                                    omit(grapherConfigComputed, ["bounds"])
+                                )}
                                 {...grapherConfigComputed}
                                 queryStr={queryStr}
                                 getGrapherInstance={setGrapherInst}