diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 26d3f2b0a474..ec3241714bc1 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -159,9 +159,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml index 4f202df1b1ea..a5cd2185f876 100644 --- a/.github/workflows/prod-build.yml +++ b/.github/workflows/prod-build.yml @@ -286,9 +286,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 9cec7d32ff9d..2764fa23b270 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -302,9 +302,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4f8c96e0af0e..1f57f034fd5a 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -193,9 +193,6 @@ jobs: # Generate sitemap index file yarn build --sitemap-index - # SSR all pages - yarn render:html - # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json @@ -240,6 +237,7 @@ jobs: run: | npm ci npm run build-redirects + npm run build-canonicals - name: Deploy Function if: ${{ ! vars.SKIP_FUNCTION }} diff --git a/cloud-function/package-lock.json b/cloud-function/package-lock.json index 9ff4f349ac23..79c05a653a31 100644 --- a/cloud-function/package-lock.json +++ b/cloud-function/package-lock.json @@ -21,6 +21,7 @@ "dotenv": "^16.0.3", "express": "^4.19.2", "http-proxy-middleware": "^3.0.0", + "on-headers": "^1.0.2", "sanitize-filename": "^1.6.3" }, "devDependencies": { @@ -28,6 +29,7 @@ "@types/accept-language-parser": "^1.5.3", "@types/http-proxy": "^1.17.10", "@types/http-server": "^0.12.1", + "@types/on-headers": "^1.0.3", "cross-env": "^7.0.3", "http-proxy": "^1.18.1", "http-server": "^14.1.1", @@ -595,6 +597,15 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz", "integrity": "sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==" }, + "node_modules/@types/on-headers": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@types/on-headers/-/on-headers-1.0.3.tgz", + "integrity": "sha512-jvGNvFo8uOL6fiBGvD4Ul4lT8mZoJ57l3h0ZN/a1oHziTTXUV3slaRcYm2K1wvvLX1fhIg9AvKykxKFt3mM+Xg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -2670,6 +2681,14 @@ "node": ">= 0.8" } }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/opener": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", diff --git a/cloud-function/package.json b/cloud-function/package.json index 2ff6b8639f34..e198f6b8dcfe 100644 --- a/cloud-function/package.json +++ b/cloud-function/package.json @@ -12,8 +12,9 @@ "build-canonicals": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-canonicals.ts", "build-redirects": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-redirects.ts", "copy-internal": "rm -rf ./src/internal && cp -R ../libs ./src/internal", + "copy-ssr": "mkdir -p ./src/internal/ssr && cp ../ssr/dist/main.js ./src/internal/ssr/", "gcp-build": "npm run build", - "prepare": "([ ! -e ../libs ] || npm run copy-internal)", + "prepare": "([ ! -e ../libs ] || npm run copy-internal) && ([ ! -e ../ssr/dist ] || npm run copy-ssr)", "proxy": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/proxy.ts", "server": "npm run build && functions-framework --target=mdnHandler", "server:watch": "nodemon --exec npm run server", @@ -39,6 +40,7 @@ "dotenv": "^16.0.3", "express": "^4.19.2", "http-proxy-middleware": "^3.0.0", + "on-headers": "^1.0.2", "sanitize-filename": "^1.6.3" }, "devDependencies": { @@ -46,6 +48,7 @@ "@types/accept-language-parser": "^1.5.3", "@types/http-proxy": "^1.17.10", "@types/http-server": "^0.12.1", + "@types/on-headers": "^1.0.3", "cross-env": "^7.0.3", "http-proxy": "^1.18.1", "http-server": "^14.1.1", diff --git a/cloud-function/src/app.ts b/cloud-function/src/app.ts index 1f7221f70d4f..926b4c2e66de 100644 --- a/cloud-function/src/app.ts +++ b/cloud-function/src/app.ts @@ -23,8 +23,11 @@ import { resolveRunnerHtml } from "./middlewares/resolve-runner-html.js"; import { proxyRunner } from "./handlers/proxy-runner.js"; import { stripForwardedHostHeaders } from "./middlewares/stripForwardedHostHeaders.js"; import { proxyPong } from "./handlers/proxy-pong.js"; +import { renderIndexHTML } from "./handlers/render-html.js"; +import { addServerTimingHeaders } from "./middlewares/server-timing.js"; const router = Router(); +router.use(addServerTimingHeaders); router.use(stripForwardedHostHeaders); router.use(redirectLeadingSlash); // MDN Plus plans. @@ -88,6 +91,7 @@ router.get( redirectTrailingSlash, redirectMovedPages, resolveIndexHTML, + renderIndexHTML, proxyContent ); router.get( @@ -96,6 +100,7 @@ router.get( redirectLocale, redirectEnforceTrailingSlash, resolveIndexHTML, + renderIndexHTML, proxyContent ); // MDN Plus, static pages, etc. @@ -106,6 +111,7 @@ router.get( redirectLocale, redirectTrailingSlash, resolveIndexHTML, + renderIndexHTML, proxyContent ); router.all("*", notFound); diff --git a/cloud-function/src/handlers/proxy-content.ts b/cloud-function/src/handlers/proxy-content.ts index 55ad0e31c793..363339f1a8c7 100644 --- a/cloud-function/src/handlers/proxy-content.ts +++ b/cloud-function/src/handlers/proxy-content.ts @@ -1,18 +1,16 @@ -/* eslint-disable n/no-unsupported-features/node-builtins */ import { createProxyMiddleware, fixRequestBody, responseInterceptor, } from "http-proxy-middleware"; -import { withContentResponseHeaders } from "../headers.js"; +import { withProxiedContentResponseHeaders } from "../headers.js"; import { Source, sourceUri } from "../env.js"; import { PROXY_TIMEOUT } from "../constants.js"; import { isLiveSampleURL } from "../utils.js"; +import { renderHTMLForContext } from "./render-html.js"; -const NOT_FOUND_PATH = "en-us/_spas/404.html"; - -let notFoundBuffer: ArrayBuffer; +import type { Request } from "express"; const target = sourceUri(Source.content); @@ -25,25 +23,18 @@ export const proxyContent = createProxyMiddleware({ selfHandleResponse: true, on: { proxyReq: fixRequestBody, - proxyRes: responseInterceptor( + proxyRes: responseInterceptor( async (responseBuffer, proxyRes, req, res) => { - withContentResponseHeaders(proxyRes, req, res); if (proxyRes.statusCode === 404 && !isLiveSampleURL(req.url ?? "")) { - const tryHtml = await fetch( - `${target}${req.url?.slice(1)}/index.html` + const html = await renderHTMLForContext( + req, + res, + `${target}${req.url?.slice(1)}/index.json` ); - if (tryHtml.ok) { - res.statusCode = 200; - res.setHeader("Content-Type", "text/html"); - return Buffer.from(await tryHtml.arrayBuffer()); - } else if (!notFoundBuffer) { - const response = await fetch(`${target}${NOT_FOUND_PATH}`); - notFoundBuffer = await response.arrayBuffer(); - } - res.setHeader("Content-Type", "text/html"); - return Buffer.from(notFoundBuffer); + return Buffer.from(html); } + withProxiedContentResponseHeaders(proxyRes, req, res); return responseBuffer; } ), diff --git a/cloud-function/src/handlers/render-html.ts b/cloud-function/src/handlers/render-html.ts new file mode 100644 index 000000000000..32034b5e2d79 --- /dev/null +++ b/cloud-function/src/handlers/render-html.ts @@ -0,0 +1,73 @@ +/* eslint-disable n/no-unsupported-features/node-builtins */ +import type { NextFunction, Request, Response } from "express"; +import type { IncomingMessage, ServerResponse } from "node:http"; + +import { captureException } from "@sentry/serverless"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import { renderHTML } from "../internal/ssr/main.js"; +import { sourceUri, Source } from "../env.js"; +import { withRenderedContentResponseHeaders } from "../headers.js"; + +const target = sourceUri(Source.content); + +export async function renderIndexHTML( + req: Request, + res: Response, + next: NextFunction +) { + if (req.url.endsWith("/index.html")) { + req.startServerTiming("totalSSR"); + const html = await renderHTMLForContext( + req, + res, + target.replace(/\/$/, "") + req.url.replace(/html$/, "json") + ); + res.send(html).end(); + } else { + next(); + } +} + +export async function renderHTMLForContext( + req: Request, + res: ServerResponse, + contextUrl: string +) { + res.setHeader("Content-Type", "text/html"); + res.setHeader("X-MDN-SSR", "true"); + let context; + + try { + req.startServerTiming("fetchJSON"); + const contextRes = await fetch(contextUrl); + req.endServerTiming("fetchJSON"); + if (!contextRes.ok) { + throw new Error(contextRes.statusText); + } + req.startServerTiming("parseText"); + const text = await contextRes.text(); + req.endServerTiming("parseText"); + req.startServerTiming("parseJSON"); + context = JSON.parse(text); + req.endServerTiming("parseJSON"); + res.statusCode = 200; + } catch { + context = { url: req.url, pageNotFound: true }; + res.statusCode = 404; + } + + try { + withRenderedContentResponseHeaders(req, res); + req.startServerTiming("renderHTML"); + const html = renderHTML(context); + req.endServerTiming("renderHTML"); + return html; + } catch (e) { + captureException(e); + res.statusCode = 500; + res.setHeader("Content-Type", "text/plain"); + return "Internal Server Error"; + } +} diff --git a/cloud-function/src/headers.ts b/cloud-function/src/headers.ts index e595fc448226..cea241f4906c 100644 --- a/cloud-function/src/headers.ts +++ b/cloud-function/src/headers.ts @@ -14,7 +14,7 @@ const NO_CACHE_VALUE = "no-store, must-revalidate"; const HASHED_REGEX = /\.[a-f0-9]{8,32}\./; -export function withContentResponseHeaders( +export function withProxiedContentResponseHeaders( proxyRes: IncomingMessage, req: IncomingMessage, res: ServerResponse @@ -56,6 +56,29 @@ export function withContentResponseHeaders( return res; } +export function withRenderedContentResponseHeaders( + req: IncomingMessage, + res: ServerResponse +) { + if (res.headersSent) { + console.warn( + `Cannot set content response headers. Headers already sent for: ${req.url}` + ); + return; + } + + const url = req.url ?? ""; + + setContentResponseHeaders((name, value) => res.setHeader(name, value), {}); + + const cacheControl = getCacheControl(res.statusCode ?? 0, url); + if (cacheControl) { + res.setHeader("Cache-Control", cacheControl); + } + + return res; +} + function getCacheControl(statusCode: number, url: string) { if ( statusCode === 404 || diff --git a/cloud-function/src/middlewares/server-timing.ts b/cloud-function/src/middlewares/server-timing.ts new file mode 100644 index 000000000000..7e3ee68588a7 --- /dev/null +++ b/cloud-function/src/middlewares/server-timing.ts @@ -0,0 +1,37 @@ +import onHeaders from "on-headers"; +import { hrtime } from "node:process"; + +import type { NextFunction, Request, Response } from "express"; + +export async function addServerTimingHeaders( + req: Request, + res: Response, + next: NextFunction +) { + const timers = new Map(); + + req.startServerTiming = (id: string) => { + timers.set(id, { start: hrtime.bigint() }); + }; + + req.endServerTiming = (id: string) => { + const start = timers.get(id)?.start; + if (start !== undefined) { + timers.set(id, { start, diff: hrtime.bigint() - start }); + } + }; + + req.startServerTiming("total"); + + onHeaders(res, () => { + const header = [...timers] + .map(([id, { start, diff }]) => { + const time = diff !== undefined ? diff : hrtime.bigint() - start; + return `${id};dur=${time / BigInt(1e6)}`; + }) + .join(", "); + res.setHeader("Server-Timing", header); + }); + + next(); +} diff --git a/cloud-function/src/types/express.d.ts b/cloud-function/src/types/express.d.ts new file mode 100644 index 000000000000..e857a198ab39 --- /dev/null +++ b/cloud-function/src/types/express.d.ts @@ -0,0 +1,6 @@ +declare namespace Express { + interface Request { + startServerTiming: (id: string) => void; + endServerTiming: (id: string) => void; + } +} diff --git a/cloud-function/tsconfig.json b/cloud-function/tsconfig.json index fae5d6bfe79b..15967df886a7 100644 --- a/cloud-function/tsconfig.json +++ b/cloud-function/tsconfig.json @@ -10,6 +10,7 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "moduleResolution": "nodenext", + "allowSyntheticDefaultImports": true, "allowUnusedLabels": false, "allowUnreachableCode": false, "exactOptionalPropertyTypes": true, @@ -20,7 +21,7 @@ "noUncheckedIndexedAccess": true, "noUnusedLocals": true, "noUnusedParameters": true, - "checkJs": true + "typeRoots": ["./node_modules/@types", "./src/types"] }, "ts-node": { "esm": true, diff --git a/package.json b/package.json index 52242fb7aeac..10286e8729f3 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "build:docs": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts -n", "build:glean": "cd client && cross-env VIRTUAL_ENV=venv glean translate src/telemetry/metrics.yaml src/telemetry/pings.yaml -f typescript -o src/telemetry/generated", "build:prepare": "yarn build:client && yarn build:ssr && yarn tool popularities && yarn tool spas && yarn tool gather-git-history && yarn tool build-robots-txt", - "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && webpack --mode=production", + "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && cross-env NODE_ENV=production webpack --mode=production", "build:sw": "cd client/pwa && yarn && yarn build:prod", "build:sw-dev": "cd client/pwa && yarn && yarn build", "check:tsc": "find . -name 'tsconfig.json' ! -wholename '**/node_modules/**' -print0 | xargs -n1 -P 2 -0 sh -c 'cd `dirname $0` && echo \"🔄 $(pwd)\" && npx tsc --noEmit && echo \"☑️ $(pwd)\" || exit 255'", @@ -56,7 +56,7 @@ "test:prepare": "yarn build:prepare && yarn build:docs && yarn render:html && yarn start:static-server", "test:testing": "yarn jest --rootDir testing", "tool": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ./tool/cli.ts", - "watch:ssr": "cd ssr && webpack --mode=production --watch" + "watch:ssr": "cd ssr && cross-env NODE_ENV=production webpack --mode=production --watch" }, "resolutions": { "http-cache-semantics": ">=4.1.1", diff --git a/ssr/webpack.config.js b/ssr/webpack.config.js index 64a4195de08d..42e7cd9204a4 100644 --- a/ssr/webpack.config.js +++ b/ssr/webpack.config.js @@ -1,6 +1,9 @@ import { fileURLToPath } from "node:url"; import nodeExternals from "webpack-node-externals"; import webpack from "webpack"; +import getClientEnvironment from "../client/config/env.js"; + +const env = getClientEnvironment(); const config = { context: fileURLToPath(new URL(".", import.meta.url)), @@ -94,6 +97,7 @@ const config = { new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1, }), + new webpack.DefinePlugin(env.stringified), ], experiments: { outputModule: true,