From 380b1f5df1b046ff04a01494ec04e990948e8fdb Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Mon, 19 Aug 2024 09:36:00 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Render=20thumbnails=20for=20grapher?= =?UTF-8?q?s=20by=20uuid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example-full | 2 +- functions/_common/grapherRenderer.ts | 26 ++++++---------- functions/_common/reusableHandlers.ts | 37 ++++++++++++++++++++++ functions/grapher/[slug].ts | 44 ++++++++++++++++++++++++--- functions/grapher/by-uuid/[uuid].ts | 33 ++++++++++++++++++-- functions/grapher/thumbnail/[slug].ts | 23 ++++++++++++-- settings/clientSettings.ts | 3 +- site/search/SearchPanel.tsx | 2 +- 8 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 functions/_common/reusableHandlers.ts diff --git a/.env.example-full b/.env.example-full index d0cb5063adf..854f02e91c5 100644 --- a/.env.example-full +++ b/.env.example-full @@ -36,7 +36,7 @@ GRAPHER_CONFIG_R2_BUCKET_PATH= # optional - for local dev set it to "devs/YOURNA OPENAI_API_KEY= -GRAPHER_DYNAMIC_THUMBNAIL_URL= # optional; can set this to https://ourworldindata.org/grapher/thumbnail to use the live thumbnail worker +GRAPHER_DYNAMIC_THUMBNAIL_URL= # optional; can set this to https://ourworldindata.org/grapher to use the live thumbnail worker # enable search (readonly) ALGOLIA_ID= # optional diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 2326d5442fd..df952a01252 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -23,6 +23,8 @@ declare global { var window: any } +export type Etag = string + const grapherBaseUrl = "https://ourworldindata.org/grapher" // Lots of defaults; these are mostly the same as they are in owid-grapher. @@ -166,17 +168,17 @@ interface FetchGrapherConfigResult { etag: string | undefined } -interface GrapherSlug { +export interface GrapherSlug { type: "slug" id: string } -interface GrapherUuid { +export interface GrapherUuid { type: "uuid" id: string } -type GrapherIdentifier = GrapherSlug | GrapherUuid +export type GrapherIdentifier = GrapherSlug | GrapherUuid export async function fetchUnparsedGrapherConfig( identifier: GrapherIdentifier, @@ -267,17 +269,14 @@ export async function fetchGrapherConfig( } async function fetchAndRenderGrapherToSvg( - slug: string, + id: GrapherIdentifier, options: ImageOptions, searchParams: URLSearchParams, env: Env ): Promise { const grapherLogger = new TimeLogger("grapher") - const grapherConfigResponse = await fetchGrapherConfig( - { type: "slug", id: slug }, - env - ) + const grapherConfigResponse = await fetchGrapherConfig(id, env) if (grapherConfigResponse.status === 404) { // we throw 404 errors instad of returning a 404 response so that the router @@ -320,20 +319,15 @@ async function fetchAndRenderGrapherToSvg( } export const fetchAndRenderGrapher = async ( - slug: string, + id: GrapherIdentifier, searchParams: URLSearchParams, outType: "png" | "svg", env: Env ) => { const options = extractOptions(searchParams) - console.log("Rendering", slug, outType, options) - const svg = await fetchAndRenderGrapherToSvg( - slug, - options, - searchParams, - env - ) + console.log("Rendering", id.id, outType, options) + const svg = await fetchAndRenderGrapherToSvg(id, options, searchParams, env) console.log("fetched svg") switch (outType) { diff --git a/functions/_common/reusableHandlers.ts b/functions/_common/reusableHandlers.ts new file mode 100644 index 00000000000..8a86c2aa280 --- /dev/null +++ b/functions/_common/reusableHandlers.ts @@ -0,0 +1,37 @@ +import { Env } from "./env.js" +import { + Etag, + GrapherIdentifier, + fetchAndRenderGrapher, +} from "./grapherRenderer.js" + +export async function handleThumbnailRequest( + id: GrapherIdentifier, + searchParams: URLSearchParams, + env: Env, + _etag: Etag, + ctx: EventContext>, + extension: "png" | "svg" +) { + const url = new URL(env.url) + const shouldCache = !url.searchParams.has("nocache") + + const cache = caches.default + console.log("Handling", env.url, ctx.request.headers.get("User-Agent")) + if (shouldCache) { + console.log("Checking cache") + const maybeCached = await cache.match(ctx.request) + console.log("Cache check result", maybeCached ? "hit" : "miss") + if (maybeCached) return maybeCached + } + const resp = await fetchAndRenderGrapher(id, searchParams, extension, env) + if (shouldCache) { + resp.headers.set("Cache-Control", "public, s-maxage=3600, max-age=3600") + ctx.waitUntil(caches.default.put(ctx.request, resp.clone())) + } else + resp.headers.set( + "Cache-Control", + "public, s-maxage=0, max-age=0, must-revalidate" + ) + return resp +} diff --git a/functions/grapher/[slug].ts b/functions/grapher/[slug].ts index 942ae68ae01..be2ed7ba0b1 100644 --- a/functions/grapher/[slug].ts +++ b/functions/grapher/[slug].ts @@ -2,18 +2,27 @@ import { Env } from "../_common/env.js" import { getOptionalRedirectForSlug, createRedirectResponse, + Etag, fetchUnparsedGrapherConfig, } from "../_common/grapherRenderer.js" import { IRequestStrict, Router, StatusError, error, cors } from "itty-router" +import { handleThumbnailRequest } from "../_common/reusableHandlers.js" const { preflight, corsify } = cors({ allowMethods: ["GET", "OPTIONS", "HEAD"], }) -const extensions = { +// We collect the possible extensions here so we can easily take them into account +// when handling redirects +export const extensions = { configJson: ".config.json", + png: ".png", + svg: ".svg", } -const router = Router({ +const router = Router< + IRequestStrict, + [URL, Env, Etag, EventContext>] +>({ before: [preflight], finally: [corsify], }) @@ -23,6 +32,30 @@ router async ({ params: { slug } }, { searchParams }, env, etag) => handleConfigRequest(slug, searchParams, env, etag) ) + .get( + `/grapher/:slug${extensions.png}`, + async ({ params: { slug } }, { searchParams }, env, etag, ctx) => + handleThumbnailRequest( + { type: "slug", id: slug }, + searchParams, + env, + etag, + ctx, + "png" + ) + ) + .get( + `/grapher/:slug${extensions.svg}`, + async ({ params: { slug } }, { searchParams }, env, etag, ctx) => + handleThumbnailRequest( + { type: "slug", id: slug }, + searchParams, + env, + etag, + ctx, + "svg" + ) + ) .get( "/grapher/:slug", async ({ params: { slug } }, { searchParams }, env) => @@ -42,7 +75,8 @@ export const onRequest: PagesFunction = async (context) => { request, url, { ...env, url }, - request.headers.get("if-none-match") + request.headers.get("if-none-match"), + context ) .catch(async (e) => { // Here we do a unified after the fact handling of 404s to check @@ -119,10 +153,10 @@ async function handleHtmlPageRequest( // In the case of the redirect, the browser will then request the new URL which will again be handled by this worker. if (grapherPageResp.status !== 200) return grapherPageResp - const openGraphThumbnailUrl = `/grapher/thumbnail/${slug}.png?imType=og${ + const openGraphThumbnailUrl = `/grapher/${slug}.png?imType=og${ url.search ? "&" + url.search.slice(1) : "" }` - const twitterThumbnailUrl = `/grapher/thumbnail/${slug}.png?imType=twitter${ + const twitterThumbnailUrl = `/grapher/${slug}.png?imType=twitter${ url.search ? "&" + url.search.slice(1) : "" }` diff --git a/functions/grapher/by-uuid/[uuid].ts b/functions/grapher/by-uuid/[uuid].ts index 9fcc75d7785..b3365175682 100644 --- a/functions/grapher/by-uuid/[uuid].ts +++ b/functions/grapher/by-uuid/[uuid].ts @@ -1,14 +1,40 @@ import { Env } from "../../_common/env.js" import { fetchGrapherConfig } from "../../_common/grapherRenderer.js" import { IRequestStrict, Router, error, StatusError } from "itty-router" +import { handleThumbnailRequest } from "../../_common/reusableHandlers.js" +import { extensions } from "../[slug].js" const router = Router() router .get( - "/grapher/by-uuid/:uuid.config.json", + `/grapher/by-uuid/:uuid${extensions.configJson}`, async ({ params: { uuid } }, { searchParams }, env, etag) => handleConfigRequest(uuid, searchParams, env, etag) ) + .get( + `/grapher/by-uuid/:uuid${extensions.png}`, + async ({ params: { uuid } }, { searchParams }, env, etag, ctx) => + handleThumbnailRequest( + { type: "uuid", id: uuid }, + searchParams, + env, + etag, + ctx, + "png" + ) + ) + .get( + `/grapher/by-uuid/:uuid${extensions.svg}`, + async ({ params: { uuid } }, { searchParams }, env, etag, ctx) => + handleThumbnailRequest( + { type: "uuid", id: uuid }, + searchParams, + env, + etag, + ctx, + "svg" + ) + ) .all("*", () => error(404, "Route not defined")) export const onRequest: PagesFunction = async (context) => { @@ -20,7 +46,8 @@ export const onRequest: PagesFunction = async (context) => { request, url, { ...env, url }, - request.headers.get("if-none-match") + request.headers.get("if-none-match"), + context ) .catch((e) => { if (e instanceof StatusError) { @@ -56,7 +83,7 @@ async function handleConfigRequest( ? "public, s-maxage=3600, max-age=0, must-revalidate" : "public, s-maxage=0, max-age=0, must-revalidate" - return new Response(JSON.stringify(grapherPageResp.grapherConfig), { + return Response.json(grapherPageResp.grapherConfig, { headers: { "content-type": "application/json", "Cache-Control": cacheControl, diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts index d92306bd67d..9b66aebfdb9 100644 --- a/functions/grapher/thumbnail/[slug].ts +++ b/functions/grapher/thumbnail/[slug].ts @@ -2,22 +2,39 @@ import { Env } from "../../_common/env.js" import { fetchAndRenderGrapher } from "../../_common/grapherRenderer.js" import { IRequestStrict, Router, error } from "itty-router" +// TODO: remove the /grapher/thumbnail route two weeks or so after the change to use /grapher/:slug.png is deployed +// We keep this around for another two weeks so that cached html pages etc can still fetch the correct thumbnail const router = Router() router .get( "/grapher/thumbnail/:slug.png", async ({ params: { slug } }, { searchParams }, env) => - fetchAndRenderGrapher(slug, searchParams, "png", env) + fetchAndRenderGrapher( + { type: "slug", id: slug }, + searchParams, + "png", + env + ) ) .get( "/grapher/thumbnail/:slug.svg", async ({ params: { slug } }, { searchParams }, env) => - fetchAndRenderGrapher(slug, searchParams, "svg", env) + fetchAndRenderGrapher( + { type: "slug", id: slug }, + searchParams, + "svg", + env + ) ) .get( "/grapher/thumbnail/:slug", async ({ params: { slug } }, { searchParams }, env) => - fetchAndRenderGrapher(slug, searchParams, "svg", env) + fetchAndRenderGrapher( + { type: "slug", id: slug }, + searchParams, + "svg", + env + ) ) .all("*", () => error(404, "Route not defined")) diff --git a/settings/clientSettings.ts b/settings/clientSettings.ts index 750d1dc4b8c..a678d27c04c 100644 --- a/settings/clientSettings.ts +++ b/settings/clientSettings.ts @@ -35,8 +35,7 @@ export const BAKED_SITE_EXPORTS_BASE_URL: string = process.env.BAKED_SITE_EXPORTS_BASE_URL ?? `${BAKED_BASE_URL}/exports` export const GRAPHER_DYNAMIC_THUMBNAIL_URL: string = - process.env.GRAPHER_DYNAMIC_THUMBNAIL_URL ?? - `${BAKED_GRAPHER_URL}/thumbnail` + process.env.GRAPHER_DYNAMIC_THUMBNAIL_URL ?? `${BAKED_GRAPHER_URL}` export const ADMIN_BASE_URL: string = process.env.ADMIN_BASE_URL ?? diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index d77c3d7fcd0..8014849c142 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -159,7 +159,7 @@ function ChartHit({ ) const queryStr = useMemo(() => getEntityQueryStr(entities), [entities]) const previewUrl = queryStr - ? `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${hit.slug}${queryStr}` + ? `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${hit.slug}.svg${queryStr}` : `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${hit.slug}.svg` useEffect(() => {