🔨 use grapher configs from R2 for thumbnail rendering

adminSiteServer/apiRouter.ts

@@ -85,6 +85,7 @@ import {
     DbInsertUser,
     FlatTagGraph,
     DbRawChartConfig,
+    R2GrapherConfigDirectory,
 } from "@ourworldindata/types"
 import { uuidv7 } from "uuidv7"
 import {
@@ -158,7 +159,6 @@ import path from "path"
 import {
     deleteGrapherConfigFromR2,
     deleteGrapherConfigFromR2ByUUID,
-    R2GrapherConfigDirectory,
     saveGrapherConfigToR2,
     saveGrapherConfigToR2ByUUID,
     getMd5HashBase64,

adminSiteServer/chartConfigR2Helpers.ts

@@ -14,6 +14,7 @@ import {
     S3Client,
 } from "@aws-sdk/client-s3"
 import { Base64String, JsonError } from "@ourworldindata/utils"
+import { R2GrapherConfigDirectory } from "@ourworldindata/types"
 import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 import { createHash } from "crypto"
 
@@ -25,10 +26,6 @@ export function getMd5HashBase64(data: string): Base64String {
         .update(data, "utf-8")
         .digest("base64") as Base64String
 }
-export enum R2GrapherConfigDirectory {
-    byUUID = "config/by-uuid",
-    publishedGrapherBySlug = "grapher/by-slug",
-}
 
 let s3Client: S3Client | undefined = undefined
 

devTools/syncGraphersToR2/syncGraphersToR2.ts

@@ -23,7 +23,6 @@ import {
     KnexReadonlyTransaction,
     knexReadonlyTransaction,
 } from "../../db/db.js"
-import { R2GrapherConfigDirectory } from "../../adminSiteServer/chartConfigR2Helpers.js"
 import {
     base64ToBytes,
     bytesToBase64,
@@ -32,6 +31,7 @@ import {
     excludeUndefined,
     HexString,
     hexToBytes,
+    R2GrapherConfigDirectory,
 } from "@ourworldindata/utils"
 import { string } from "ts-pattern/dist/patterns.js"
 import { chunk, take } from "lodash"

functions/README.md

@@ -10,6 +10,8 @@ Pages Functions are very similar to Cloudflare Workers; however they will always
 Pages Functions use file-based routing, which means that the file `grapher/[slug].ts` will serve routes like `/grapher/child-mortality`.
 In addition, there's a [`_routes.json`](../_routes.json) file that specifies which routes are to be served dynamically.
 
+Inside a file-based route we sometimes use an instance of itty-router to decide on the exact functionality to provide (e.g. png vs svg generation)
+
 ## Development
 
 1. Copy `.dev.vars.example` to `.dev.vars` and fill in the required variables.
@@ -28,15 +30,17 @@ Note: compatibility dates between local development, production and preview envi
 
 ## Testing on Fondation staging sites vs Cloudfare previews
 
-`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].owid-staging.pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details.
+We have two cloudflare projects set up that you can deploy previews to. `owid` which is also where our production deployment runs, and `owid-staging`. Currently, `owid` is configured to require authentication while `owid-staging` is accessible from the internet without any kind of auth.
+
+`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].[PROJECT].pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details.
 
 ### Rationale
 
 A custom staging site is available at http://staging-site-[BRANCH] upon pushing your branch (see ops > templates > lxc-manager > staging-create). This site is served by `wrangler` (see ops > templates > owid-site-staging > grapher-refresh.sh). `wrangler` is helpful for testing the functions locally (and possibly for some debugging scenarios on staging servers), but is still not the closest match to the production Cloudflare environment.
 
 When it comes to testing functions in a production-like environment, Cloudflare previews are recommended.
 
-Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].owid-staging.pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file. Instead, they use the [Cloudflare dashboard to configure environment variables](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site.
+Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].[PROJECT].pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file, but they do take the `wrangler.toml` file into account for environment variables. For secrets, they use the [values set via the Cloudflare dashboard](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site.
 
 This proximity of configurations in the Cloudflare dashboard makes spotting differences between production and preview environments easier - and is one of the reason of using Cloudflare previews in the same project (owid) over using a new project specific to staging.
 

functions/_common/grapherRenderer.ts

@@ -1,5 +1,10 @@
-import { Grapher, GrapherInterface } from "@ourworldindata/grapher"
-import { Bounds, deserializeJSONFromHTML } from "@ourworldindata/utils"
+import { Grapher } from "@ourworldindata/grapher"
+import {
+    Bounds,
+    excludeUndefined,
+    GrapherInterface,
+    R2GrapherConfigDirectory,
+} from "@ourworldindata/utils"
 import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
 import { TimeLogger } from "./timeLogger"
 import { png } from "itty-router"
@@ -143,19 +148,33 @@ async function fetchAndRenderGrapherToSvg({
 }) {
     const grapherLogger = new TimeLogger("grapher")
 
-    // Fetch grapher config and extract it from the HTML
-    const grapherConfig: GrapherInterface = await env.ASSETS.fetch(
-        new URL(`/grapher/${slug}`, env.url)
-    )
-        .then((r) => (r.ok ? r : Promise.reject("Failed to load grapher page")))
-        .then((r) => r.text())
-        .then((html) => deserializeJSONFromHTML(html))
+    const url = new URL(`/grapher/${slug}`, env.url)
+    const slugOnly = url.pathname.split("/").pop()
 
-    if (!grapherConfig) {
-        throw new Error("Could not find grapher config")
+    // The top level directory is either the bucket path (should be set in dev environments and production)
+    // or the branch name on preview staging environments
+    console.log("branch", env.CF_PAGES_BRANCH)
+    const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_PATH
+        ? [env.GRAPHER_CONFIG_R2_BUCKET_PATH]
+        : ["by-branch", env.CF_PAGES_BRANCH]
+
+    const key = excludeUndefined([
+        ...topLevelDirectory,
+        R2GrapherConfigDirectory.publishedGrapherBySlug,
+        `${slugOnly}.json`,
+    ]).join("/")
+
+    console.log("fetching grapher config from this key", key)
+
+    // Fetch grapher config
+    const fetchResponse = await env.r2ChartConfigs.get(key)
+
+    if (!fetchResponse) {
+        return null
     }
 
-    grapherLogger.log("fetchGrapherConfig")
+    const grapherConfig: GrapherInterface = await fetchResponse.json()
+    console.log("grapher title", grapherConfig.title)
 
     const bounds = new Bounds(0, 0, options.svgWidth, options.svgHeight)
     const grapher = new Grapher({
@@ -206,6 +225,10 @@ export const fetchAndRenderGrapher = async (
         env,
     })
 
+    if (!svg) {
+        return new Response("Not found", { status: 404 })
+    }
+
     switch (outType) {
         case "png":
             return png(await renderSvgToPng(svg, options))

functions/grapher/thumbnail/[slug].ts

@@ -5,7 +5,12 @@ export interface Env {
     ASSETS: {
         fetch: typeof fetch
     }
+    r2ChartConfigs: {
+        get: (url: string) => Promise<R2ObjectBody>
+    }
     url: URL
+    GRAPHER_CONFIG_R2_BUCKET_PATH: string
+    CF_PAGES_BRANCH: string
     ENV: string
 }
 

package.json

@@ -26,7 +26,7 @@
         "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts",
         "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js",
         "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx",
-        "startLocalCloudflareFunctions": "wrangler pages dev",
+        "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage",
         "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js",
         "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents",
         "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"",

packages/@ourworldindata/types/src/domainTypes/Various.ts

@@ -65,3 +65,8 @@ export class JsonError extends Error {
 export interface QueryParams {
     [key: string]: string | undefined
 }
+
+export enum R2GrapherConfigDirectory {
+    byUUID = "config/by-uuid",
+    publishedGrapherBySlug = "grapher/by-slug",
+}

packages/@ourworldindata/types/src/index.ts

@@ -18,6 +18,7 @@ export {
     type RawPageview,
     type UserCountryInformation,
     type QueryParams,
+    R2GrapherConfigDirectory,
 } from "./domainTypes/Various.js"
 export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js"
 export {

wrangler.toml

@@ -13,11 +13,26 @@ MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1"
 ENV = "preview"
 
+[[r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs-staging"
+
 # Overrides for CF production deployment
 [env.production]
 compatibility_date = "2024-04-29"
 
+[[env.production.r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs"
+
 [env.production.vars]
 ENV = "production"
 MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C5JJW19PS"
+GRAPHER_CONFIG_R2_BUCKET_PATH = "v1"
+
+
+[[env.preview.r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs-staging"
+