feat(ssr): render in Cloud Function, not in build

.github/workflows/dev-build.yml

@@ -159,9 +159,6 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
-          # SSR all pages
-          yarn render:html
-
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

.github/workflows/prod-build.yml

@@ -286,9 +286,6 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
-          # SSR all pages
-          yarn render:html
-
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

.github/workflows/stage-build.yml

@@ -302,9 +302,6 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
-          # SSR all pages
-          yarn render:html
-
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json

.github/workflows/test-build.yml

@@ -193,9 +193,6 @@ jobs:
           # Generate sitemap index file
           yarn build --sitemap-index
 
-          # SSR all pages
-          yarn render:html
-
           # Generate whatsdeployed files.
           yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json
@@ -240,6 +237,7 @@ jobs:
         run: |
           npm ci
           npm run build-redirects
+          npm run build-canonicals
 
       - name: Deploy Function
         if: ${{ ! vars.SKIP_FUNCTION }}

cloud-function/package.json

@@ -12,8 +12,9 @@
     "build-canonicals": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-canonicals.ts",
     "build-redirects": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/build-redirects.ts",
     "copy-internal": "rm -rf ./src/internal && cp -R ../libs ./src/internal",
+    "copy-ssr": "mkdir -p ./src/internal/ssr && cp ../ssr/dist/main.js ./src/internal/ssr/",
     "gcp-build": "npm run build",
-    "prepare": "([ ! -e ../libs ] || npm run copy-internal)",
+    "prepare": "([ ! -e ../libs ] || npm run copy-internal) && ([ ! -e ../ssr/dist ] || npm run copy-ssr)",
     "proxy": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node src/proxy.ts",
     "server": "npm run build && functions-framework --target=mdnHandler",
     "server:watch": "nodemon --exec npm run server",
@@ -39,13 +40,15 @@
     "dotenv": "^16.0.3",
     "express": "^4.19.2",
     "http-proxy-middleware": "^3.0.0",
+    "on-headers": "^1.0.2",
     "sanitize-filename": "^1.6.3"
   },
   "devDependencies": {
     "@swc/core": "^1.3.38",
     "@types/accept-language-parser": "^1.5.3",
     "@types/http-proxy": "^1.17.10",
     "@types/http-server": "^0.12.1",
+    "@types/on-headers": "^1.0.3",
     "cross-env": "^7.0.3",
     "http-proxy": "^1.18.1",
     "http-server": "^14.1.1",

cloud-function/src/app.ts

@@ -23,8 +23,11 @@ import { resolveRunnerHtml } from "./middlewares/resolve-runner-html.js";
 import { proxyRunner } from "./handlers/proxy-runner.js";
 import { stripForwardedHostHeaders } from "./middlewares/stripForwardedHostHeaders.js";
 import { proxyPong } from "./handlers/proxy-pong.js";
+import { renderIndexHTML } from "./handlers/render-html.js";
+import { addServerTimingHeaders } from "./middlewares/server-timing.js";
 
 const router = Router();
+router.use(addServerTimingHeaders);
 router.use(stripForwardedHostHeaders);
 router.use(redirectLeadingSlash);
 // MDN Plus plans.
@@ -88,6 +91,7 @@ router.get(
   redirectTrailingSlash,
   redirectMovedPages,
   resolveIndexHTML,
+  renderIndexHTML,
   proxyContent
 );
 router.get(
@@ -96,6 +100,7 @@ router.get(
   redirectLocale,
   redirectEnforceTrailingSlash,
   resolveIndexHTML,
+  renderIndexHTML,
   proxyContent
 );
 // MDN Plus, static pages, etc.
@@ -106,6 +111,7 @@ router.get(
   redirectLocale,
   redirectTrailingSlash,
   resolveIndexHTML,
+  renderIndexHTML,
   proxyContent
 );
 router.all("*", notFound);

cloud-function/src/handlers/proxy-content.ts

@@ -1,18 +1,16 @@
-/* eslint-disable n/no-unsupported-features/node-builtins */
 import {
   createProxyMiddleware,
   fixRequestBody,
   responseInterceptor,
 } from "http-proxy-middleware";
 
-import { withContentResponseHeaders } from "../headers.js";
+import { withProxiedContentResponseHeaders } from "../headers.js";
 import { Source, sourceUri } from "../env.js";
 import { PROXY_TIMEOUT } from "../constants.js";
 import { isLiveSampleURL } from "../utils.js";
+import { renderHTMLForContext } from "./render-html.js";
 
-const NOT_FOUND_PATH = "en-us/_spas/404.html";
-
-let notFoundBuffer: ArrayBuffer;
+import type { Request } from "express";
 
 const target = sourceUri(Source.content);
 
@@ -25,25 +23,18 @@ export const proxyContent = createProxyMiddleware({
   selfHandleResponse: true,
   on: {
     proxyReq: fixRequestBody,
-    proxyRes: responseInterceptor(
+    proxyRes: responseInterceptor<Request>(
       async (responseBuffer, proxyRes, req, res) => {
-        withContentResponseHeaders(proxyRes, req, res);
         if (proxyRes.statusCode === 404 && !isLiveSampleURL(req.url ?? "")) {
-          const tryHtml = await fetch(
-            `${target}${req.url?.slice(1)}/index.html`
+          const html = await renderHTMLForContext(
+            req,
+            res,
+            `${target}${req.url?.slice(1)}/index.json`
           );
-          if (tryHtml.ok) {
-            res.statusCode = 200;
-            res.setHeader("Content-Type", "text/html");
-            return Buffer.from(await tryHtml.arrayBuffer());
-          } else if (!notFoundBuffer) {
-            const response = await fetch(`${target}${NOT_FOUND_PATH}`);
-            notFoundBuffer = await response.arrayBuffer();
-          }
-          res.setHeader("Content-Type", "text/html");
-          return Buffer.from(notFoundBuffer);
+          return Buffer.from(html);
         }
 
+        withProxiedContentResponseHeaders(proxyRes, req, res);
         return responseBuffer;
       }
     ),

cloud-function/src/handlers/render-html.ts

@@ -0,0 +1,73 @@
+/* eslint-disable n/no-unsupported-features/node-builtins */
+import type { NextFunction, Request, Response } from "express";
+import type { IncomingMessage, ServerResponse } from "node:http";
+
+import { captureException } from "@sentry/serverless";
+
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { renderHTML } from "../internal/ssr/main.js";
+import { sourceUri, Source } from "../env.js";
+import { withRenderedContentResponseHeaders } from "../headers.js";
+
+const target = sourceUri(Source.content);
+
+export async function renderIndexHTML(
+  req: Request,
+  res: Response,
+  next: NextFunction
+) {
+  if (req.url.endsWith("/index.html")) {
+    req.startServerTiming("totalSSR");
+    const html = await renderHTMLForContext(
+      req,
+      res,
+      target.replace(/\/$/, "") + req.url.replace(/html$/, "json")
+    );
+    res.send(html).end();
+  } else {
+    next();
+  }
+}
+
+export async function renderHTMLForContext(
+  req: Request,
+  res: ServerResponse<IncomingMessage>,
+  contextUrl: string
+) {
+  res.setHeader("Content-Type", "text/html");
+  res.setHeader("X-MDN-SSR", "true");
+  let context;
+
+  try {
+    req.startServerTiming("fetchJSON");
+    const contextRes = await fetch(contextUrl);
+    req.endServerTiming("fetchJSON");
+    if (!contextRes.ok) {
+      throw new Error(contextRes.statusText);
+    }
+    req.startServerTiming("parseText");
+    const text = await contextRes.text();
+    req.endServerTiming("parseText");
+    req.startServerTiming("parseJSON");
+    context = JSON.parse(text);
+    req.endServerTiming("parseJSON");
+    res.statusCode = 200;
+  } catch {
+    context = { url: req.url, pageNotFound: true };
+    res.statusCode = 404;
+  }
+
+  try {
+    withRenderedContentResponseHeaders(req, res);
+    req.startServerTiming("renderHTML");
+    const html = renderHTML(context);
+    req.endServerTiming("renderHTML");
+    return html;
+  } catch (e) {
+    captureException(e);
+    res.statusCode = 500;
+    res.setHeader("Content-Type", "text/plain");
+    return "Internal Server Error";
+  }
+}

cloud-function/src/headers.ts

@@ -14,7 +14,7 @@ const NO_CACHE_VALUE = "no-store, must-revalidate";
 
 const HASHED_REGEX = /\.[a-f0-9]{8,32}\./;
 
-export function withContentResponseHeaders(
+export function withProxiedContentResponseHeaders(
   proxyRes: IncomingMessage,
   req: IncomingMessage,
   res: ServerResponse<IncomingMessage>
@@ -56,6 +56,29 @@ export function withContentResponseHeaders(
   return res;
 }
 
+export function withRenderedContentResponseHeaders(
+  req: IncomingMessage,
+  res: ServerResponse<IncomingMessage>
+) {
+  if (res.headersSent) {
+    console.warn(
+      `Cannot set content response headers. Headers already sent for: ${req.url}`
+    );
+    return;
+  }
+
+  const url = req.url ?? "";
+
+  setContentResponseHeaders((name, value) => res.setHeader(name, value), {});
+
+  const cacheControl = getCacheControl(res.statusCode ?? 0, url);
+  if (cacheControl) {
+    res.setHeader("Cache-Control", cacheControl);
+  }
+
+  return res;
+}
+
 function getCacheControl(statusCode: number, url: string) {
   if (
     statusCode === 404 ||

cloud-function/src/middlewares/server-timing.ts

@@ -0,0 +1,37 @@
+import onHeaders from "on-headers";
+import { hrtime } from "node:process";
+
+import type { NextFunction, Request, Response } from "express";
+
+export async function addServerTimingHeaders(
+  req: Request,
+  res: Response,
+  next: NextFunction
+) {
+  const timers = new Map<string, { start: bigint; diff?: bigint }>();
+
+  req.startServerTiming = (id: string) => {
+    timers.set(id, { start: hrtime.bigint() });
+  };
+
+  req.endServerTiming = (id: string) => {
+    const start = timers.get(id)?.start;
+    if (start !== undefined) {
+      timers.set(id, { start, diff: hrtime.bigint() - start });
+    }
+  };
+
+  req.startServerTiming("total");
+
+  onHeaders(res, () => {
+    const header = [...timers]
+      .map(([id, { start, diff }]) => {
+        const time = diff !== undefined ? diff : hrtime.bigint() - start;
+        return `${id};dur=${time / BigInt(1e6)}`;
+      })
+      .join(", ");
+    res.setHeader("Server-Timing", header);
+  });
+
+  next();
+}

cloud-function/src/types/express.d.ts

@@ -0,0 +1,6 @@
+declare namespace Express {
+  interface Request {
+    startServerTiming: (id: string) => void;
+    endServerTiming: (id: string) => void;
+  }
+}

cloud-function/tsconfig.json

@@ -10,6 +10,7 @@
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
     "moduleResolution": "nodenext",
+    "allowSyntheticDefaultImports": true,
     "allowUnusedLabels": false,
     "allowUnreachableCode": false,
     "exactOptionalPropertyTypes": true,
@@ -20,7 +21,7 @@
     "noUncheckedIndexedAccess": true,
     "noUnusedLocals": true,
     "noUnusedParameters": true,
-    "checkJs": true
+    "typeRoots": ["./node_modules/@types", "./src/types"]
   },
   "ts-node": {
     "esm": true,