Client: Improve SEO

client/src/components/Lighthouse.tsx

@@ -69,7 +69,7 @@ const Description: React.FC = _props => {
         <Divider />
         <p>
           The performance figures could look promising but at the same time prompting a
-          question what will happen when the functionality is extended and the React
+          question what will happen when the functionality is extended and the client
           codebase grows. Some considerations on this subject can be found <a
           href="https://winwiz1.github.io/crisp-react/docs/benchmarks/PERFORMANCE.html#future-considerations"
           target="_blank" rel="noopener noreferrer">here</a>. Additionally, it's important

client/src/components/NameLookup.tsx

@@ -173,7 +173,7 @@ const NameLookupContent: React.FC = _props => {
         description={pageDescription}
       />
       <Container text textAlign="justified">
-        <Header as="h3">Hello from NameLookup</Header>
+        <Header as="h3">Hello from NameLookup component</Header>
         <p>
           <code>NameLookup</code> is used by <code>first.tsx</code> which has been
           set as an 'entry point' of the 'first' script bundle by <code>
@@ -189,6 +189,11 @@ const NameLookupContent: React.FC = _props => {
           required to access the cloud service are held by the backend.
         </p>
         }
+        { CF_PAGES && <p>
+          Since there is no backend in Jamstack builds, <code>NameLookup</code> has
+          to query the API endpoint directly.
+        </p>
+        }
         <Divider horizontal css={cssDivider}>API</Divider>
         <div css={cssFlexContainer}>
           <Segment compact basic css={cssFlexItem}>

client/src/entrypoints/first.tsx

@@ -26,12 +26,16 @@ import { isServer, getHistory } from "../utils/postprocess/misc";
 import "../css/app.css";
 import "../css/app.less";
 
+// If the first SPA is called 'first' then the regex
+// will match '/first' and '/first.html';
+const regexPath = new RegExp(`^/${SPAs.getRedirectName()}(\.html)?$`);
+
 const First: React.FC = _props => {
 
   const catchAll = () => {
     const path = window.location.pathname.toLowerCase();
 
-    if (path === ("/" + SPAs.getRedirectName())) {
+    if (regexPath.test(path)) {
       return <Overview/>
     }
 

client/src/utils/misc.ts

@@ -17,5 +17,6 @@ export const getCanonical = (pagePath?: string): string|undefined => {
 
 export const getTitle = (pageTitle?: string): string => {
   // eslint-disable-next-line no-extra-boolean-cast
-  return !!pageTitle? `${SPAs.appTitle} - ${pageTitle}` : SPAs.appTitle;
+  const ret = !!pageTitle? `${SPAs.appTitle} - ${pageTitle}` : SPAs.appTitle;
+  return ret + (CF_PAGES? " (Jamstack build)" : " (Full stack build)");
 }

client/src/utils/postprocess/postProcess.ts

@@ -86,22 +86,6 @@ Please check the 4-step sequence (provided in the comments at the top of each en
     console.log("Finished SSR post-processing")
   }
 
-  if (process.env.CF_PAGES) {
-    const writeFile = promisify(fs.writeFile);
-    const redirectName = require("../../../config/spa.config").getRedirectName();
-    const stapleName = "index";
-    const redirectFile = path.join(workDir, "_redirects");
-
-    if (redirectName.toLowerCase() !== stapleName) {
-      try {
-        await writeFile(redirectFile, `/ ${redirectName} 301`);
-      } catch (e) {
-        console.error(`Failed to create redirect file, exception: ${e}`);
-        process.exit(1);
-      }
-    }
-  }
-
   try {
     await postProcessCSS();
   } catch (e) {

deployments/cloudflare/worker-fullstack.js

@@ -0,0 +1,93 @@
+/********* Start Worker Customisation *********/
+
+// Replace 'crisp-react.winwiz1.com' with the custom
+// domain or subdomain used for full stack deployment.
+const siteMap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://crisp-react.winwiz1.com</loc>
+  </url>
+  <url>
+    <loc>https://crisp-react.winwiz1.com/a</loc>
+  </url>
+  <url>
+    <loc>https://crisp-react.winwiz1.com/namelookup</loc>
+  </url>
+  <url>
+    <loc>https://crisp-react.winwiz1.com/lighthouse</loc>
+  </url>
+  <url>
+    <loc>https://crisp-react.winwiz1.com/second</loc>
+  </url>
+</urlset>
+`;
+
+// List landing (e.g. index) pages
+// of your SPAs here along with the
+// root path "/".
+const spaPaths = [
+  "/",
+  "/first",
+  "/second",
+];
+
+********* End Worker Customisation *********/
+
+const bots = [
+  "googlebot",
+  "bingbot",
+  "yahoo",
+  "applebot",
+  "yandex",
+  "baidu",
+];
+
+class ElementHandler {
+  element(element) {
+    element?.replace('<div id="app-root"></div>', {html: true});
+  }
+
+  comments(comment) {
+  }
+
+  text(text) {
+  }
+}
+
+addEventListener("fetch", event => {
+  event.respondWith(handleRequest(event.request))
+})
+
+/**
+ * Respond to the request
+ * @param {Request} request
+ */
+async function handleRequest(req) {
+  const parsedUrl = new URL(req.url);
+  const path = parsedUrl.pathname.toLowerCase();
+  const lastIdx = path.lastIndexOf(".");
+  const extensionLess = lastIdx === -1;
+  const extension = path.substring(lastIdx);
+  const userAgent = (req.headers.get("User-Agent") || "")?.toLowerCase() ?? "";
+
+  if (path === "/sitemap.xml") {
+    return new Response(
+      siteMap,
+      {
+        headers: {"content-type": "text/xml;charset=UTF-8"},
+      }
+    );
+  }
+
+  if ((extension === ".html" || extensionLess === true) &&
+    (bots.some(bot => userAgent.indexOf(bot) !== -1) ||
+    !spaPaths.includes(path))) {
+    const res = await fetch(req);
+    return new HTMLRewriter().on(
+	  "div[id='app-root']",
+	  new ElementHandler()
+	).transform(res);
+  } else {
+   return fetch(req);
+  }
+}

deployments/cloudflare/worker-jamstack.js

@@ -0,0 +1,154 @@
+/********* Start Worker Customisation *********/
+
+// Replace 'jamstack.winwiz1.com' with the custom
+// domain or subdomain used for Jamstack deployment.
+const siteMap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://jamstack.winwiz1.com</loc>
+  </url>
+  <url>
+    <loc>https://jamstack.winwiz1.com/a</loc>
+  </url>
+  <url>
+    <loc>https://jamstack.winwiz1.com/namelookup</loc>
+  </url>
+  <url>
+    <loc>https://jamstack.winwiz1.com/lighthouse</loc>
+  </url>
+  <url>
+    <loc>https://jamstack.winwiz1.com/second</loc>
+  </url>
+</urlset>
+`;
+
+// Replace 'crisp-react.pages.dev' with the subdomain
+// created by Cloudflare Pages. This subdomain is
+// referred to as 'per-project subdomain' in README.
+const pagesDomain = "https://crisp-react.pages.dev";
+
+// Adjust to ensure that for each SPA, its landing
+// (e.g. index) page and the internal pages are listed.
+const redirects = {
+  // Request for the root page has to be redirected
+  // to the SPA that has 'redirect' flag set to 'true'
+  "/":              "/first",
+
+  // The internal "/a" page belongs
+  // to the SPA named 'first'
+  "/a":            "/first",
+
+  // The internal "/namelookup" page belongs
+  // to the SPA called 'first'
+  "/namelookup":   "/first",
+
+  // The internal "/lighthouse" page belongs
+  // to the SPA called 'first'
+  "/lighthouse":   "/first",
+
+  // The landing page "/first" belongs
+  // to the SPA called 'first'.
+  "/first":        "/first",
+
+  // The landing page "/second" belongs
+  // to the SPA called 'second'.
+  "/second":        "/second",
+
+  // There are no internal pages that belong
+  // to the SPA called "second". Otherwise those
+  // pages would have been listed here.
+};
+
+/********* End Worker Customisation *********/
+
+const spaPaths = ["/"];
+
+for (const [key, value] of Object.entries(redirects)) {
+  if (key === value) {
+    spaPaths.push(key);
+  }
+};
+
+const robotsTxt = `User-agent: *
+Allow: /
+`;
+
+const bots = [
+  "googlebot",
+  "bingbot",
+  "yahoo",
+  "applebot",
+  "yandex",
+  "baidu",
+];
+
+class ElementHandler {
+  element(element) {
+    element?.replace('<div id="app-root"></div>', {html: true});
+  }
+
+  comments(comment) {
+  }
+
+  text(text) {
+  }
+}
+
+function getRedirectPath(path) {
+  if (path in redirects) {
+    return redirects[path];
+  }
+  return path;
+}
+
+addEventListener("fetch", (event) => {
+  event.respondWith(
+    handleRequest(event.request).catch(
+      (err) => new Response(err.stack, { status: 500 })
+    )
+  );
+});
+
+/**
+ * Respond to the request
+ * @param {Request} request
+ */
+async function handleRequest(req) {
+  const parsedUrl = new URL(req.url);
+  const path = parsedUrl.pathname.toLowerCase();
+  const lastIdx = path.lastIndexOf(".");
+  const extensionLess = lastIdx === -1;
+  const extension = path.substring(lastIdx);
+  const userAgent = (req.headers.get("User-Agent") || "")?.toLowerCase() ?? "";
+  const urlToFetch = pagesDomain + getRedirectPath(path);
+
+  if (path === "/sitemap.xml") {
+    return new Response(
+      siteMap,
+      {
+        headers: {"content-type": "text/xml;charset=UTF-8"},
+      }
+    );
+  }
+
+  if (path === "/robots.txt") {
+    return new Response(
+      robotsTxt,
+      {
+        headers: {"content-type": "text/plain;charset=UTF-8"},
+      }
+    );
+  }
+
+  if ((extension === ".html" || extensionLess === true) &&
+    (bots.some(bot => userAgent.indexOf(bot) !== -1) ||
+    !spaPaths.includes(path))) {
+    const res = await fetch(urlToFetch, req);
+    return new HTMLRewriter().on(
+      "div[id='app-root']",
+      new ElementHandler()
+    ).transform(res);
+  } else {
+    return fetch(urlToFetch, req);
+  }
+}

docs/ProjectHighlights.md

@@ -88,11 +88,5 @@ This arrangement brings a security benefit: The clients running inside a browser
 
 * Seamless debugging. Debug a minified/obfuscated, compressed production bundle and put breakpoints in its TypeScript code using both VS Code and Chrome DevTools. Development build debugging: put breakpoints in the client and backend code and debug both simultaneously using a single instance of VS Code.
 
-* Sample websites.
-    * [Demo - Full stack](https://crisp-react.winwiz1.com). Automated build performed by Heroku.
-    * [Demo - Jamstack](https://crisp-react.pages.dev). Automated build performed by Cloudflare Pages.
-    * [Production](https://virusquery.com). Based on Crisp React.
-
-
 Back to the [README](https://github.com/winwiz1/crisp-react#project-highlights).
 

docs/Scenarios.md

@@ -59,6 +59,11 @@ To finish, remove the breakpoint and stop the running debugging configuration (u
 #### Lint client
 Terminal: `yarn lint`
 ### Backend Usage Scenarios
+The same Express server is used in production and debugging.
+
+This is done to keep the difference between production and debugging environments to the minimum. If the difference is significant then it could be difficult to debug customer issues. You might be unable to even reproduce it.
+
+In some of the debugging scenarios another process, webpack-dev-server, is automatically started in background to facilitate debugging and recompile the client code as you type changes. But importantly the browser you use for debugging 'doesn't know' about that because it 'can see' Express only.
 #### Build backend in production mode
 Open a command prompt in the directory containing the workspace file `crisp-react.code-workspace` .<br/>
 Execute command: `yarn build:prod`.<br/>