feat(explorers): allow referencing indicators by catalog path (#3748)

adminSiteServer/adminRouter.tsx

@@ -304,15 +304,18 @@ getPlainRouteWithROTransaction(
         if (slug === DefaultNewExplorerSlug)
             return renderExplorerPage(
                 new ExplorerProgram(DefaultNewExplorerSlug, ""),
-                knex
+                knex,
+                { isPreviewing: true }
             )
         if (
             !slug ||
             !fs.existsSync(explorerAdminServer.absoluteFolderPath + filename)
         )
             return `File not found`
         const explorer = await explorerAdminServer.getExplorerFromFile(filename)
-        const explorerPage = await renderExplorerPage(explorer, knex)
+        const explorerPage = await renderExplorerPage(explorer, knex, {
+            isPreviewing: true,
+        })
 
         return res.send(explorerPage)
     }

adminSiteServer/mockSiteRouter.tsx

@@ -189,8 +189,10 @@ getPlainRouteWithROTransaction(
         const program =
             await explorerAdminServer.getExplorerFromSlug(explorerSlug)
         const explorerPage = await renderExplorerPage(program, trx, {
-            explorerUrlMigrationId: migrationId,
-            baseQueryStr,
+            urlMigrationSpec: {
+                explorerUrlMigrationId: migrationId,
+                baseQueryStr,
+            },
         })
         res.send(explorerPage)
     }

baker/ExplorerBaker.tsx

@@ -6,6 +6,119 @@ import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.
 import { explorerRedirectTable } from "../explorerAdminServer/ExplorerRedirects.js"
 import { renderExplorerPage } from "./siteRenderers.js"
 import * as db from "../db/db.js"
+import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
+import { ExplorerGrammar } from "../explorer/ExplorerGrammar.js"
+import {
+    CoreTable,
+    ErrorValueTypes,
+    isNotErrorValueOrEmptyCell,
+} from "@ourworldindata/core-table"
+import { ColumnGrammar } from "../explorer/ColumnGrammar.js"
+import { ColumnTypeNames } from "@ourworldindata/types"
+
+export const transformExplorerProgramToResolveCatalogPaths = async (
+    program: ExplorerProgram,
+    knex: db.KnexReadonlyTransaction
+): Promise<{
+    program: ExplorerProgram
+    unresolvedCatalogPaths?: Set<string>
+}> => {
+    const { decisionMatrix } = program
+    const { requiredCatalogPaths } = decisionMatrix
+
+    if (requiredCatalogPaths.size === 0) return { program }
+
+    const catalogPathToIndicatorIdMap = await getVariableIdsByCatalogPath(
+        [...requiredCatalogPaths],
+        knex
+    )
+    const unresolvedCatalogPaths = new Set(
+        [...requiredCatalogPaths].filter(
+            (path) => !catalogPathToIndicatorIdMap.get(path)
+        )
+    )
+
+    const colSlugsToUpdate =
+        decisionMatrix.allColumnsWithIndicatorIdsOrCatalogPaths.map(
+            (col) => col.slug
+        )
+    // In the decision matrix table, replace any catalog paths with their corresponding indicator ids
+    // If a catalog path is not found, it will be left as is
+    const newDecisionMatrixTable =
+        decisionMatrix.tableWithOriginalColumnNames.replaceCells(
+            colSlugsToUpdate,
+            (val) => {
+                if (typeof val === "string") {
+                    const vals = val.split(" ")
+                    const updatedVals = vals.map(
+                        (val) =>
+                            catalogPathToIndicatorIdMap.get(val)?.toString() ??
+                            val
+                    )
+                    return updatedVals.join(" ")
+                }
+                return val
+            }
+        )
+
+    // Write the result to the "graphers" block
+    const grapherBlockLine = program.getRowMatchingWords(
+        ExplorerGrammar.graphers.keyword
+    )
+    if (grapherBlockLine === -1)
+        throw new Error(
+            `"graphers" block not found in explorer ${program.slug}`
+        )
+    const newProgram = program.updateBlock(
+        grapherBlockLine,
+        newDecisionMatrixTable.toMatrix()
+    )
+
+    // Next, we also need to update the "columns" block of the explorer
+    program.columnDefsByTableSlug.forEach((_columnDefs, tableSlug) => {
+        const lineNoInProgram = newProgram.getRowMatchingWords(
+            ExplorerGrammar.columns.keyword,
+            tableSlug
+        )
+        // This should, in theory, never happen because columnDefsByTableSlug gets generated from such a block
+        if (lineNoInProgram === -1)
+            throw new Error(
+                `Column defs not found for explorer ${program.slug} and table ${tableSlug}`
+            )
+        const columnDefTable = new CoreTable(
+            newProgram.getBlock(lineNoInProgram)
+        )
+        const newColumnDefsTable = columnDefTable.combineColumns(
+            [
+                ColumnGrammar.variableId.keyword,
+                ColumnGrammar.catalogPath.keyword,
+            ],
+            {
+                slug: ColumnGrammar.variableId.keyword,
+                type: ColumnTypeNames.Integer,
+            },
+            (row) => {
+                const variableId = row[ColumnGrammar.variableId.keyword]
+                if (isNotErrorValueOrEmptyCell(variableId)) return variableId
+
+                const catalogPath = row[ColumnGrammar.catalogPath.keyword]
+                if (
+                    isNotErrorValueOrEmptyCell(catalogPath) &&
+                    typeof catalogPath === "string"
+                ) {
+                    return (
+                        catalogPathToIndicatorIdMap.get(catalogPath) ??
+                        ErrorValueTypes.NoMatchingVariableId
+                    )
+                }
+                return ErrorValueTypes.NoMatchingVariableId
+            }
+        )
+        newProgram.updateBlock(lineNoInProgram, newColumnDefsTable.toMatrix())
+    })
+
+    return { program: newProgram.clone, unresolvedCatalogPaths }
+}
 
 export const bakeAllPublishedExplorers = async (
     outputFolder: string,
@@ -58,8 +171,10 @@ export const bakeAllExplorerRedirects = async (
             )
         }
         const html = await renderExplorerPage(program, knex, {
-            explorerUrlMigrationId: migrationId,
-            baseQueryStr,
+            urlMigrationSpec: {
+                explorerUrlMigrationId: migrationId,
+                baseQueryStr,
+            },
         })
         await write(path.join(outputFolder, `${redirectPath}.html`), html)
     }

baker/siteRenderers.tsx

@@ -91,6 +91,7 @@ import {
     getAndLoadGdocBySlug,
     getAndLoadGdocById,
 } from "../db/model/Gdoc/GdocFactory.js"
+import { transformExplorerProgramToResolveCatalogPaths } from "./ExplorerBaker.js"
 
 export const renderToHtmlPage = (element: any) =>
     `<!doctype html>${ReactDOMServer.renderToStaticMarkup(element)}`
@@ -691,12 +692,34 @@ export const renderReusableBlock = async (
     return cheerioEl("body").html() ?? undefined
 }
 
+interface ExplorerRenderOpts {
+    urlMigrationSpec?: ExplorerPageUrlMigrationSpec
+    isPreviewing?: boolean
+}
+
 export const renderExplorerPage = async (
     program: ExplorerProgram,
     knex: KnexReadonlyTransaction,
-    urlMigrationSpec?: ExplorerPageUrlMigrationSpec
+    opts?: ExplorerRenderOpts
 ) => {
-    const { requiredGrapherIds, requiredVariableIds } = program.decisionMatrix
+    const transformResult = await transformExplorerProgramToResolveCatalogPaths(
+        program,
+        knex
+    )
+    const { program: transformedProgram, unresolvedCatalogPaths } =
+        transformResult
+    if (unresolvedCatalogPaths?.size) {
+        const errMessage = new JsonError(
+            `${unresolvedCatalogPaths.size} catalog paths cannot be found for explorer ${transformedProgram.slug}: ${[...unresolvedCatalogPaths].join(", ")}.`
+        )
+        if (opts?.isPreviewing) console.error(errMessage)
+        else void logErrorAndMaybeSendToBugsnag(errMessage)
+    }
+
+    // This needs to run after transformExplorerProgramToResolveCatalogPaths, so that the catalog paths
+    // have already been resolved and all the required grapher and variable IDs are available
+    const { requiredGrapherIds, requiredVariableIds } =
+        transformedProgram.decisionMatrix
 
     type ChartRow = { id: number; config: string }
     let grapherConfigRows: ChartRow[] = []
@@ -726,7 +749,7 @@ export const renderExplorerPage = async (
         if (missingIds.length > 0) {
             void logErrorAndMaybeSendToBugsnag(
                 new JsonError(
-                    `Referenced variable IDs do not exist in the database for explorer ${program.slug}: ${missingIds.join(", ")}.`
+                    `Referenced variable IDs do not exist in the database for explorer ${transformedProgram.slug}: ${missingIds.join(", ")}.`
                 )
             )
         }
@@ -758,10 +781,13 @@ export const renderExplorerPage = async (
             return mergePartialGrapherConfigs(etlConfig, adminConfig)
         })
 
-    const wpContent = program.wpBlockId
+    const wpContent = transformedProgram.wpBlockId
         ? await renderReusableBlock(
-              await getBlockContentFromSnapshot(knex, program.wpBlockId),
-              program.wpBlockId,
+              await getBlockContentFromSnapshot(
+                  knex,
+                  transformedProgram.wpBlockId
+              ),
+              transformedProgram.wpBlockId,
               knex
           )
         : undefined
@@ -772,10 +798,11 @@ export const renderExplorerPage = async (
             <ExplorerPage
                 grapherConfigs={grapherConfigs}
                 partialGrapherConfigs={partialGrapherConfigs}
-                program={program}
+                program={transformedProgram}
                 wpContent={wpContent}
                 baseUrl={BAKED_BASE_URL}
-                urlMigrationSpec={urlMigrationSpec}
+                urlMigrationSpec={opts?.urlMigrationSpec}
+                isPreviewing={opts?.isPreviewing}
             />
         )
     )

db/model/Variable.ts

@@ -19,6 +19,7 @@ import {
     DimensionProperty,
     GrapherInterface,
     DbRawVariable,
+    VariablesTableName,
 } from "@ourworldindata/types"
 import { knexRaw } from "../db.js"
 
@@ -578,3 +579,23 @@ export interface VariableResultView {
     table: string
     shortName: string
 }
+
+export const getVariableIdsByCatalogPath = async (
+    catalogPaths: string[],
+    knex: db.KnexReadonlyTransaction
+): Promise<Map<string, number | null>> => {
+    const rows: Pick<DbRawVariable, "id" | "catalogPath">[] = await knex
+        .select("id", "catalogPath")
+        .from(VariablesTableName)
+        .whereIn("catalogPath", catalogPaths)
+
+    const rowsByPath = _.keyBy(rows, "catalogPath")
+
+    // `rowsByPath` only contains the rows that were found, so we need to create
+    // a map where all keys from `catalogPaths` are present, and set the value to
+    // undefined if no row was found for that catalog path.
+    return new Map(
+        // Sort for good measure and determinism.
+        catalogPaths.sort().map((path) => [path, rowsByPath[path]?.id ?? null])
+    )
+}

explorer/ColumnGrammar.ts

@@ -8,6 +8,7 @@ import { ToleranceStrategy } from "@ourworldindata/utils"
 import {
     BooleanCellDef,
     EnumCellDef,
+    EtlPathCellDef,
     Grammar,
     IntegerCellDef,
     NumericCellDef,
@@ -22,6 +23,11 @@ export const ColumnGrammar: Grammar = {
         keyword: "variableId",
         description: "Numerical variable ID",
     },
+    catalogPath: {
+        ...EtlPathCellDef,
+        keyword: "catalogPath",
+        description: "Catalog path to the etl indicator",
+    },
     slug: {
         ...SlugDeclarationCellDef,
         keyword: "slug",

explorer/Explorer.scss

@@ -9,6 +9,11 @@ $explorer-padding: 0.5rem;
     width: 100%;
     position: relative;
     padding: $explorer-padding;
+
+    .admin-only-locally-edited-checkbox {
+        position: absolute;
+        right: 5px;
+    }
 }
 
 html.IsInIframe #ExplorerContainer {
@@ -17,6 +22,10 @@ html.IsInIframe #ExplorerContainer {
     max-height: none;
     // leave some padding for shadows
     padding: 3px;
+
+    .admin-only-locally-edited-checkbox {
+        display: none;
+    }
 }
 
 .ExplorerHeaderBox {