Skip to content

Commit

Permalink
✨ render csv and metadata.json in CF worker for grapher charts
Browse files Browse the repository at this point in the history
  • Loading branch information
danyx23 committed Aug 28, 2024
1 parent 8e9dbef commit e7f2fcb
Show file tree
Hide file tree
Showing 15 changed files with 1,026 additions and 73 deletions.
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -140,4 +140,4 @@
"restart": true
},
]
}
}
334 changes: 325 additions & 9 deletions functions/_common/grapherRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,16 @@ import {
excludeUndefined,
GrapherInterface,
R2GrapherConfigDirectory,
OwidColumnDef,
getCitationShort,
getAttributionFragmentsFromVariable,
getCitationLong,
getLastUpdatedFromVariable,
OwidTableSlugs,
getNextUpdateFromVariable,
} from "@ourworldindata/utils"
import { OwidOrigin } from "@ourworldindata/types"
import { constructReadme } from "./readmeTools"
import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
import { TimeLogger } from "./timeLogger"
import { png, StatusError } from "itty-router"
Expand All @@ -17,6 +26,7 @@ import LatoMedium from "../_common/fonts/LatoLatin-Medium.ttf.bin"
import LatoBold from "../_common/fonts/LatoLatin-Bold.ttf.bin"
import PlayfairSemiBold from "../_common/fonts/PlayfairDisplayLatin-SemiBold.ttf.bin"
import { Env } from "./env.js"
import { fromPairs } from "lodash"

declare global {
// eslint-disable-next-line no-var
Expand Down Expand Up @@ -267,16 +277,13 @@ export async function fetchGrapherConfig(
etag: fetchResponse.headers.get("etag"),
}
}

async function fetchAndRenderGrapherToSvg(
id: GrapherIdentifier,
async function initGrapher(
identifier: GrapherIdentifier,
options: ImageOptions,
searchParams: URLSearchParams,
env: Env
): Promise<string> {
const grapherLogger = new TimeLogger("grapher")

const grapherConfigResponse = await fetchGrapherConfig(id, env)
): Promise<Grapher> {
const grapherConfigResponse = await fetchGrapherConfig(identifier, env)

if (grapherConfigResponse.status === 404) {
// we throw 404 errors instad of returning a 404 response so that the router
Expand All @@ -296,10 +303,319 @@ async function fetchAndRenderGrapherToSvg(
})
grapher.shouldIncludeDetailsInStaticExport = options.details

grapherLogger.log("grapherInit")
return grapher
}

export async function fetchMetadataForGrapher(
identifier: GrapherIdentifier,
env: Env,
searchParams?: URLSearchParams
) {
console.log("Initializing grapher")
const grapher = await initGrapher(
identifier,
TWITTER_OPTIONS,
searchParams ?? new URLSearchParams(""),
env
)

await grapher.downloadLegacyDataFromOwidVariableIds()

//const useShortNames = searchParams.get("useColumnShortNames") === "true"

const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
OwidTableSlugs.time,
OwidTableSlugs.entityColor,
OwidTableSlugs.entityName,
OwidTableSlugs.entityCode,
OwidTableSlugs.year,
OwidTableSlugs.day,
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)
const useShortNames = searchParams.get("useColumnShortNames") === "true"
console.log("useShortNames", useShortNames)

const columns: [
string,
{
title: string
titleProducer: string
titleVariant: string
descriptionShort: string
descriptionFromProducer: string
descriptionKey: string[]
descriptionProcessing: string
shortUnit: string
unit: string
timespan: string
tolerance: number
type: string
conversionFactor: number
owidVariableId: number
catalogPath: string
sources: Partial<
Pick<
OwidOrigin,
| "attribution"
| "attributionShort"
| "description"
| "urlDownload"
| "urlMain"
>
>[]
shortName: string
},
][] = grapher.inputTable.getColumns(columnsToGet).map((col) => {
console.log("mapping col", col.name)
const {
descriptionShort,
descriptionFromProducer,

Check warning on line 378 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'descriptionFromProducer' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 378 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'descriptionFromProducer' is assigned a value but never used. Allowed unused vars must match /^_/u
descriptionKey,
descriptionProcessing,
additionalInfo,
shortUnit,
unit,
timespan,
tolerance,
type,
display,

Check warning on line 387 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'display' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 387 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'display' is assigned a value but never used. Allowed unused vars must match /^_/u
presentation,

Check warning on line 388 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'presentation' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 388 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'presentation' is assigned a value but never used. Allowed unused vars must match /^_/u
origins,
sourceLink,
sourceName,
catalogPath,

Check warning on line 392 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'catalogPath' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 392 in functions/_common/grapherRenderer.ts

View workflow job for this annotation

GitHub Actions / eslint

'catalogPath' is assigned a value but never used. Allowed unused vars must match /^_/u
owidVariableId,
shortName,
} = col.def as OwidColumnDef
const lastUpdated = getLastUpdatedFromVariable(col.def)
const nextUpdate = getNextUpdateFromVariable(col.def)

let consensedOrigins:
| Partial<
Pick<
OwidOrigin,
| "attribution"
| "attributionShort"
| "description"
| "urlDownload"
| "urlMain"
>
>[]
| undefined = origins?.map((origin) => {
const {
attribution,
attributionShort,
description,
citationFull,
urlDownload,
urlMain,
dateAccessed,
} = origin
return {
attribution,
attributionShort,
description,
urlDownload,
urlMain,
dateAccessed,
citationFull,
}
})

if (!consensedOrigins || consensedOrigins.length === 0) {
consensedOrigins = [
{
attribution: sourceName,
urlMain: sourceLink,
},
]
}

const def = col.def as OwidColumnDef

const citationShort = getCitationShort(
def.origins,
getAttributionFragmentsFromVariable(def),
def.owidProcessingLevel
)

const citationLong = getCitationLong(
col.titlePublicOrDisplayName,
def.origins ?? [],
col.source ?? {},
getAttributionFragmentsFromVariable(def),
def.presentation?.attributionShort,
def.presentation?.titleVariant,
def.owidProcessingLevel,
undefined
)

const titleShort = col.titlePublicOrDisplayName.title
const attributionShort = col.titlePublicOrDisplayName.attributionShort
const titleVariant = col.titlePublicOrDisplayName.titleVariant
const attributionString =
attributionShort && titleVariant
? `${attributionShort}${titleVariant}`
: attributionShort || titleVariant
const titleModifier = attributionString ? ` - ${attributionString}` : ""
const titleLong = `${col.titlePublicOrDisplayName.title}${titleModifier}`

return [
useShortNames ? shortName : col.name,
{
titleShort,
titleLong,
descriptionShort,
descriptionKey,
descriptionProcessing,
shortUnit,
unit,
timespan,
tolerance,
type,
conversionFactor: col.display?.conversionFactor,
owidVariableId,
shortName,
additionalInfo,
lastUpdated,
nextUpdate,
citationShort,
citationLong,
fullMetadata: `https://api.ourworldindata.org/v1/indicators/${owidVariableId}.metadata.json`,
},
]
})

const fullMetadata = {
chart: {
title: grapher.title,
subtitle: grapher.subtitle,
note: grapher.note,
xAxisLabel: grapher.xAxis.label,
yAxisLabel: grapher.yAxis.label,
citation: grapher.sourcesLine,
originalChartUrl: grapher.canonicalUrl,
selection: grapher.selectedEntityNames,
},
columns: fromPairs(columns),
}

return Response.json(fullMetadata)
}

export async function fetchZipForGrapher(
identifier: GrapherIdentifier,
env: Env,
searchParams?: URLSearchParams
) {
const grapher = await initGrapher(
identifier,
TWITTER_OPTIONS,
searchParams ?? new URLSearchParams(""),
env
)
await grapher.downloadLegacyDataFromOwidVariableIds()
const defs = grapher.inputTable
.getColumns(grapher.inputTable.columnNames)
.map((col) => col.def)
const table =
searchParams.get("csvType") === "filtered"
? grapher.transformedTable
: grapher.inputTable
const json = JSON.stringify(defs)
const zip = new JSZip()
zip.file("metadata.json", json)
zip.file("data.csv", table.toPrettyCsv())
const content = await zip.generateAsync({ type: "blob" })
return new Response(content, {
headers: {
"Content-Type": "application/zip",
},
})
}

export async function fetchCsvForGrapher(
identifier: GrapherIdentifier,
env: Env,
searchParams?: URLSearchParams
) {
const grapher = await initGrapher(
identifier,
TWITTER_OPTIONS,
searchParams ?? new URLSearchParams(""),
env
)
await grapher.downloadLegacyDataFromOwidVariableIds()
const useShortNames = searchParams.get("useColumnShortNames") === "true"
const table =
searchParams.get("csvType") === "filtered"
? grapher.transformedTable
: grapher.inputTable
return new Response(table.toPrettyCsv(useShortNames), {
headers: {
"Content-Type": "text/csv",
},
})
}

export async function fetchReadmeForGrapher(
identifier: GrapherIdentifier,
env: Env,
searchParams?: URLSearchParams
) {
console.log("Initializing grapher")
const grapher = await initGrapher(
identifier,
TWITTER_OPTIONS,
searchParams ?? new URLSearchParams(""),
env
)

await grapher.downloadLegacyDataFromOwidVariableIds()

const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
OwidTableSlugs.time,
OwidTableSlugs.entityColor,
OwidTableSlugs.entityName,
OwidTableSlugs.entityCode,
OwidTableSlugs.year,
OwidTableSlugs.day,
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)

const columns = grapher.inputTable.getColumns(columnsToGet)

const readme = constructReadme(grapher, columns)
return new Response(readme, {
headers: {
"Content-Type": "text/markdown",
},
})
}
async function fetchAndRenderGrapherToSvg(
identifier: GrapherIdentifier,
options: ImageOptions,
searchParams: URLSearchParams,
env: Env
) {
const grapherLogger = new TimeLogger("grapher")
const grapher = await initGrapher(identifier, options, searchParams, env)

grapherLogger.log("initGrapher")
const promises = []
promises.push(grapher.downloadLegacyDataFromOwidVariableIds())

if (options.details && grapher.detailsOrderedByReference.length) {
promises.push(
await fetch("https://ourworldindata.org/dods.json")
Expand Down
Loading

0 comments on commit e7f2fcb

Please sign in to comment.