Cache-bust dynamic configuration files and theme CSS

package.json

@@ -113,6 +113,7 @@
     "ngx-skeleton-loader": "^7.0.0",
     "ngx-sortablejs": "^11.1.0",
     "ngx-ui-switch": "^14.1.0",
+    "node-html-parser": "^7.0.1",
     "nouislider": "^15.8.1",
     "pem": "1.14.8",
     "reflect-metadata": "^0.2.2",

server.ts

@@ -53,6 +53,7 @@ import { ServerAppModule } from './src/main.server';
 import { buildAppConfig } from './src/config/config.server';
 import { APP_CONFIG, AppConfig } from './src/config/app-config.interface';
 import { extendEnvironmentWithAppConfig } from './src/config/config.util';
+import { ServerHashedFileMapping } from './src/modules/dynamic-hash/hashed-file-mapping.server';
 import { logStartupMessage } from './startup-message';
 import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model';
 
@@ -68,7 +69,11 @@ const indexHtml = join(DIST_FOLDER, 'index.html');
 
 const cookieParser = require('cookie-parser');
 
-const appConfig: AppConfig = buildAppConfig(join(DIST_FOLDER, 'assets/config.json'));
+const configJson = join(DIST_FOLDER, 'assets/config.json');
+const hashedFileMapping = new ServerHashedFileMapping(DIST_FOLDER, 'index.html');
+const appConfig: AppConfig = buildAppConfig(configJson, hashedFileMapping);
+hashedFileMapping.addThemeStyles();
+hashedFileMapping.save();
 
 // cache of SSR pages for known bots, only enabled in production mode
 let botCache: LRU<string, any>;
@@ -261,7 +266,7 @@ function ngApp(req, res) {
  */
 function serverSideRender(req, res, sendToUser: boolean = true) {
   // Render the page via SSR (server side rendering)
-  res.render(indexHtml, {
+  res.render(hashedFileMapping.resolve(indexHtml), {
     req,
     res,
     preboot: environment.universal.preboot,
@@ -308,7 +313,7 @@ function serverSideRender(req, res, sendToUser: boolean = true) {
  * @param res current response
  */
 function clientSideRender(req, res) {
-  res.sendFile(indexHtml);
+  res.sendFile(hashedFileMapping.resolve(indexHtml));
 }
 
 

src/app/app.module.ts

@@ -10,6 +10,8 @@ import { MetaReducer, StoreModule, USER_PROVIDED_META_REDUCERS } from '@ngrx/sto
 import { TranslateModule } from '@ngx-translate/core';
 import { ScrollToModule } from '@nicky-lenaers/ngx-scroll-to';
 import { DYNAMIC_MATCHER_PROVIDERS } from '@ng-dynamic-forms/core';
+import { HashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping';
+import { BrowserHashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping.browser';
 
 import { AppRoutingModule } from './app-routing.module';
 import { AppComponent } from './app.component';
@@ -110,6 +112,10 @@ const PROVIDERS = [
     useClass: DspaceRestInterceptor,
     multi: true
   },
+  {
+    provide: HashedFileMapping,
+    useClass: BrowserHashedFileMapping,
+  },
   // register the dynamic matcher used by form. MUST be provided by the app module
   ...DYNAMIC_MATCHER_PROVIDERS,
 ];

src/app/shared/theme-support/theme.service.spec.ts

@@ -1,6 +1,7 @@
 import { of as observableOf } from 'rxjs';
 import { TestBed } from '@angular/core/testing';
 import { provideMockActions } from '@ngrx/effects/testing';
+import { HashedFileMapping } from '../../../modules/dynamic-hash/hashed-file-mapping';
 import { LinkService } from '../../core/cache/builders/link.service';
 import { hot } from 'jasmine-marbles';
 import { SetThemeAction } from './theme.actions';
@@ -48,6 +49,10 @@ class MockLinkService {
   }
 }
 
+const mockHashedFileMapping = {
+  resolve: (path: string) => path,
+};
+
 describe('ThemeService', () => {
   let themeService: ThemeService;
   let linkService: LinkService;
@@ -101,6 +106,10 @@ describe('ThemeService', () => {
         { provide: DSpaceObjectDataService, useValue: mockDsoService },
         { provide: Router, useValue: new RouterMock() },
         { provide: ConfigurationDataService, useValue: configurationService },
+        {
+          provide: HashedFileMapping,
+          useValue: mockHashedFileMapping,
+        },
       ]
     });
 
@@ -414,6 +423,10 @@ describe('ThemeService', () => {
           { provide: DSpaceObjectDataService, useValue: mockDsoService },
           { provide: Router, useValue: new RouterMock() },
           { provide: ConfigurationDataService, useValue: configurationService },
+          {
+            provide: HashedFileMapping,
+            useValue: mockHashedFileMapping,
+          },
         ]
       });
 

src/app/shared/theme-support/theme.service.ts

@@ -1,6 +1,7 @@
 import { Injectable, Inject, Injector } from '@angular/core';
 import { Store, createFeatureSelector, createSelector, select } from '@ngrx/store';
 import { BehaviorSubject, EMPTY, Observable, of as observableOf, from, concatMap } from 'rxjs';
+import { HashedFileMapping } from '../../../modules/dynamic-hash/hashed-file-mapping';
 import { ThemeState } from './theme.reducer';
 import { SetThemeAction, ThemeActionTypes } from './theme.actions';
 import { defaultIfEmpty, expand, filter, map, switchMap, take, toArray } from 'rxjs/operators';
@@ -54,6 +55,7 @@ export class ThemeService {
     @Inject(GET_THEME_CONFIG_FOR_FACTORY) private gtcf: (str) => ThemeConfig,
     private router: Router,
     @Inject(DOCUMENT) private document: any,
+    private hashedFileMapping: HashedFileMapping,
   ) {
     // Create objects from the theme configs in the environment file
     this.themes = environment.themes.map((themeConfig: ThemeConfig) => themeFactory(themeConfig, injector));
@@ -179,7 +181,10 @@ export class ThemeService {
     link.setAttribute('rel', 'stylesheet');
     link.setAttribute('type', 'text/css');
     link.setAttribute('class', 'theme-css');
-    link.setAttribute('href', `${encodeURIComponent(themeName)}-theme.css`);
+    link.setAttribute(
+      'href',
+      this.hashedFileMapping.resolve(`${encodeURIComponent(themeName)}-theme.css`),
+    );
     // wait for the new css to download before removing the old one to prevent a
     // flash of unstyled content
     link.onload = () => {

src/config/config.server.ts

@@ -2,6 +2,7 @@ import { red, blue, green, bold } from 'colors';
 import { existsSync, readFileSync, writeFileSync } from 'fs';
 import { load } from 'js-yaml';
 import { join } from 'path';
+import { ServerHashedFileMapping } from '../modules/dynamic-hash/hashed-file-mapping.server';
 
 import { AppConfig } from './app-config.interface';
 import { Config } from './config.interface';
@@ -159,6 +160,7 @@ const buildBaseUrl = (config: ServerConfig): void => {
   ].join('');
 };
 
+
 /**
  * Build app config with the following chain of override.
  *
@@ -169,7 +171,7 @@ const buildBaseUrl = (config: ServerConfig): void => {
  * @param destConfigPath optional path to save config file
  * @returns app config
  */
-export const buildAppConfig = (destConfigPath?: string): AppConfig => {
+export const buildAppConfig = (destConfigPath?: string, mapping?: ServerHashedFileMapping): AppConfig => {
   // start with default app config
   const appConfig: AppConfig = new DefaultAppConfig();
 
@@ -237,7 +239,12 @@ export const buildAppConfig = (destConfigPath?: string): AppConfig => {
   buildBaseUrl(appConfig.rest);
 
   if (isNotEmpty(destConfigPath)) {
-    writeFileSync(destConfigPath, JSON.stringify(appConfig, null, 2));
+    const content = JSON.stringify(appConfig, null, 2);
+
+    writeFileSync(destConfigPath, content);
+    if (mapping !== undefined) {
+      mapping.add(destConfigPath, content);
+    }
 
     console.log(`Angular ${bold('config.json')} file generated correctly at ${bold(destConfigPath)} \n`);
   }

src/main.browser.ts

@@ -3,14 +3,15 @@ import 'reflect-metadata';
 import 'core-js/es/reflect';
 
 import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
-
 import { BrowserAppModule } from './modules/app/browser-app.module';
 
 import { environment } from './environments/environment';
 import { AppConfig } from './config/app-config.interface';
 import { extendEnvironmentWithAppConfig } from './config/config.util';
 import { enableProdMode } from '@angular/core';
+import { BrowserHashedFileMapping } from './modules/dynamic-hash/hashed-file-mapping.browser';
 
+const hashedFileMapping = new BrowserHashedFileMapping(document);
 const bootstrap = () => platformBrowserDynamic()
   .bootstrapModule(BrowserAppModule, {});
 
@@ -32,7 +33,7 @@ const main = () => {
     return bootstrap();
   } else {
     // Configuration must be fetched explicitly
-    return fetch('assets/config.json')
+    return fetch(hashedFileMapping.resolve('assets/config.json'))
       .then((response) => response.json())
       .then((appConfig: AppConfig) => {
         // extend environment with app config for browser when not prerendered

src/modules/dynamic-hash/hashed-file-mapping.browser.ts

@@ -0,0 +1,45 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+import { DOCUMENT } from '@angular/common';
+import {
+  Inject,
+  Injectable,
+  Optional,
+} from '@angular/core';
+import isObject from 'lodash/isObject';
+import isString from 'lodash/isString';
+import { hasValue } from '../../app/shared/empty.util';
+import {
+  HashedFileMapping,
+  ID,
+} from './hashed-file-mapping';
+
+/**
+ * Client-side implementation of {@link HashedFileMapping}.
+ * Reads out the mapping from index.html before the app is bootstrapped.
+ * Afterwards, {@link resolve} can be used to grab the latest file.
+ */
+@Injectable()
+export class BrowserHashedFileMapping extends HashedFileMapping {
+  constructor(
+    @Optional() @Inject(DOCUMENT) protected document: any,
+  ) {
+    super();
+    const element = document?.querySelector(`script#${ID}`);
+
+    if (hasValue(element?.textContent)) {
+      const mapping = JSON.parse(element.textContent);
+
+      if (isObject(mapping)) {
+        Object.entries(mapping)
+              .filter(([key, value]) => isString(key) && isString(value))
+              .forEach(([plainPath, hashPath]) => this.map.set(plainPath, hashPath));
+      }
+    }
+  }
+}

src/modules/dynamic-hash/hashed-file-mapping.server.ts

@@ -0,0 +1,138 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+import crypto from 'crypto';
+import {
+  readFileSync,
+  rmSync,
+  writeFileSync,
+  copyFileSync,
+  existsSync,
+} from 'fs';
+import glob from 'glob';
+import { parse } from 'node-html-parser';
+import {
+  extname,
+  join,
+  relative,
+} from 'path';
+import zlib from 'zlib';
+import {
+  HashedFileMapping,
+  ID,
+} from './hashed-file-mapping';
+
+/**
+ * Server-side implementation of {@link HashedFileMapping}.
+ * Registers dynamically hashed files and stores them in index.html for the browser to use.
+ */
+export class ServerHashedFileMapping extends HashedFileMapping {
+  public readonly indexPath: string;
+  private readonly indexContent: string;
+
+  constructor(
+    private readonly root: string,
+    file: string,
+  ) {
+    super();
+    this.root = join(root, '');
+    this.indexPath = join(root, file);
+    this.indexContent = readFileSync(this.indexPath).toString();
+  }
+
+  /**
+   * Add a new file to the mapping by an absolute path (within the root directory).
+   * If {@link content} is provided, the {@link path} itself does not have to exist.
+   * Otherwise, it is read out from the original path.
+   * The original path is never overwritten.
+   */
+  add(path: string, content?: string, compress = false): string {
+    if (content === undefined) {
+      content = readFileSync(path).toString();
+    }
+
+    // remove previous files
+    const ext = extname(path);
+    glob.GlobSync(path.replace(`${ext}`, `.*${ext}*`))
+        .found
+        .forEach(p => rmSync(p));
+
+    // hash the content
+    const hash = crypto.createHash('md5')
+                       .update(content)
+                       .digest('hex');
+
+    // add the hash to the path
+    const hashPath = path.replace(`${ext}`, `.${hash}${ext}`);
+
+    // store it in the mapping
+    this.map.set(path, hashPath);
+
+    // write the file
+    writeFileSync(hashPath, content);
+
+    if (compress) {
+      // write the file as .br
+      zlib.brotliCompress(content, (err, compressed) => {
+        if (err) {
+          throw new Error('Brotli compress failed');
+        } else {
+          writeFileSync(hashPath + '.br', compressed);
+        }
+      });
+
+      // write the file as .gz
+      zlib.gzip(content, (err, compressed) => {
+        if (err) {
+          throw new Error('Gzip compress failed');
+        } else {
+          writeFileSync(hashPath + '.gz', compressed);
+        }
+      });
+    }
+
+    return hashPath;
+  }
+
+  addThemeStyles() {
+    glob.GlobSync(`${this.root}/*-theme.css`)
+        .found
+        .forEach(p => {
+          const hp = this.add(p);
+          this.ensureCompressedFilesAssumingUnchangedContent(p, hp, '.br');
+          this.ensureCompressedFilesAssumingUnchangedContent(p, hp, '.gz');
+        });
+  }
+
+  private ensureCompressedFilesAssumingUnchangedContent(path: string, hashedPath: string, compression: string) {
+    const compressedPath = `${path}${compression}`;
+    const compressedHashedPath = `${hashedPath}${compression}`;
+
+    if (existsSync(compressedPath) && !existsSync(compressedHashedPath)) {
+      copyFileSync(compressedPath, compressedHashedPath);
+    }
+  }
+
+  /**
+   * Save the mapping as JSON in the index file.
+   * The updated index file itself is hashed as well, and must be sent {@link resolve}.
+   */
+  save(): void {
+    const out = Array.from(this.map.entries())
+                     .reduce((object, [plain, hashed]) => {
+                       object[relative(this.root, plain)] = relative(this.root, hashed);
+                       return object;
+                     }, {});
+
+    let root = parse(this.indexContent);
+    root.querySelector(`script#${ID}`)?.remove();
+    root.querySelector('head')
+        .appendChild(`<script id="${ID}" type="application/json">${JSON.stringify(out)}</script>` as any);
+
+    this.add(this.indexPath, root.toString());
+  }
+}