Update openhtmltopdf. Add visual regression test script (#166)

* Update openhtmltopdf. Add visual regression test script

* Add Myriad fonts

build.gradle

@@ -23,10 +23,10 @@ dependencies {
   implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0'
   implementation 'com.github.librepdf:openpdf:1.2.10'
   implementation 'org.apache.pdfbox:pdfbox:2.0.30'
-  implementation 'com.openhtmltopdf:openhtmltopdf-core:0.0.1-RC17'
-  implementation 'com.openhtmltopdf:openhtmltopdf-pdfbox:0.0.1-RC17'
-  implementation 'com.openhtmltopdf:openhtmltopdf-svg-support:0.0.1-RC17'
-  implementation 'org.apache.xmlgraphics:batik-transcoder:1.17' // shade older 1.10 provided by openhtmltopdf-svg-support to fix CVEs
+  implementation 'com.openhtmltopdf:openhtmltopdf-core:1.0.10'
+  implementation 'com.openhtmltopdf:openhtmltopdf-pdfbox:1.0.10'
+  implementation 'com.openhtmltopdf:openhtmltopdf-svg-support:1.0.10'
+  implementation 'org.apache.xmlgraphics:batik-transcoder:1.17' // shade older 1.14 provided by openhtmltopdf-svg-support to fix CVEs
   implementation 'org.apache.commons:commons-lang3:3.7'
   implementation 'commons-io:commons-io:2.7'
   implementation 'org.jsoup:jsoup:1.16.2'

src/main/java/uk/gov/beis/els/configuration/WebMvcConfiguration.java

@@ -60,6 +60,21 @@ public void registerFonts() {
 
       LOGGER.info("Arial-bold loaded {}", arialBoldLoaded);
 
+      boolean myriadProLoaded = GraphicsEnvironment.getLocalGraphicsEnvironment().registerFont(
+          Font.createFont(Font.TRUETYPE_FONT, new ClassPathResource("fonts/myriad-pro.ttf").getInputStream()));
+
+      LOGGER.info("Myriad Pro loaded {}", myriadProLoaded);
+
+      boolean myriadProBoldLoaded = GraphicsEnvironment.getLocalGraphicsEnvironment().registerFont(
+          Font.createFont(Font.TRUETYPE_FONT, new ClassPathResource("fonts/myriad-pro-bold.ttf").getInputStream()));
+
+      LOGGER.info("Myriad Pro Bold loaded {}", myriadProBoldLoaded);
+
+      boolean myriadProSemiBoldLoaded = GraphicsEnvironment.getLocalGraphicsEnvironment().registerFont(
+          Font.createFont(Font.TRUETYPE_FONT, new ClassPathResource("fonts/myriad-pro-semibold.ttf").getInputStream()));
+
+      LOGGER.info("Myriad Pro Semi Bold loaded {}", myriadProSemiBoldLoaded);
+
       LOGGER.info("Available system fonts are:");
       Arrays.stream(GraphicsEnvironment.getLocalGraphicsEnvironment().getAvailableFontFamilyNames()).forEach(f ->
           LOGGER.info("Font: {}", f)

src/main/resources/labels/html-wrapper.html

@@ -44,6 +44,21 @@
         src: url('fonts/arial.ttf');
       }
 
+      @font-face {
+        font-family: Myriad Pro;
+        src: url('fonts/myriad-pro.ttf');
+      }
+
+      @font-face {
+        font-family: MyriadPro-Bold;
+        src: url('fonts/myriad-pro-bold.ttf');
+      }
+
+      @font-face {
+        font-family: MyriadPro-Semibold;
+        src: url('fonts/myriad-pro-semibold.ttf');
+      }
+
     </style>
     <title id="html-title"></title>
   </head>

src/test/visual-regression/Readme.md

@@ -0,0 +1,32 @@
+# Visual regression test script
+
+Quick script to generate all energy labels, internet arrows and fiches, form 2 different deployments via the API. Then diff the images see changes between the deployments to catch any regression issues. 
+
+## Process
+The `fetch-all-images.js` script will:
+- Fetch and parse the OpenAPI file from the specified environment
+- Loop over each endpoint and generate the PDF and PNG versions of each label, and save to disk
+- Convert PDFs to PNG. We can only diff PNGs, so in order to check the PDF output hasn't changed either we need to then convert the PDFs to PNG.
+
+Once this has been run twice to create an old and new set of images, the `compare-images.js` script will:
+- Load both versions of each PNG and find pixels that are different between the two
+- For those with an amount of differing pixels over the given threshold, a composite diff images is created. Showing the diff, the old image and the new image, highlighting the areas that are different.
+
+## Instructions
+
+### 1. Collect baseline images
+1. Create a directory to store the baseline images.
+2. In `fetch-all-images.js`, set the `baseUrl` to point to an existing baseline deployment (e.g. the staging env) and set `folderName` to the folder you just created. 
+3. Run `fetch-all-iamges.js`. Your folder should now have a load of images.
+
+### 2. Collect new images
+1. Create a directory to store the new images.
+2. In `fetch-all-images.js`, set the `baseUrl` to point to your new deployment (e.g. localhost) and set `folderName` to the folder you just created.
+3. Run `fetch-all-iamges.js`. Your folder should now have a load of images. 
+
+### 4. Run diff
+1. Create a directory to store the visual diff output.
+2. In `compare-images.js` set the `oldDir`, `newDir` and `diffDir` fields to the 3 directories you created.
+3. Run `compare-images.js`.
+4. Review visual diff output for changes and action as needed.
+

src/test/visual-regression/compare-images.js

@@ -0,0 +1,34 @@
+import pixelmatch from "pixelmatch"
+import fs from "fs"
+import {PNG} from "pngjs";
+import combineImage from 'combine-image';
+
+const oldDir = "old";
+const newDir = "new";
+const diffDir = "diff";
+
+for (const imgFile of fs.readdirSync(`./${oldDir}`)) {
+  if (imgFile.endsWith(".pdf")) continue; // ignore pdf, we use the converted image
+
+  const imgOld = PNG.sync.read(fs.readFileSync(`./${oldDir}/${imgFile}`));
+  const imgNew = PNG.sync.read(fs.readFileSync(`./${newDir}/${imgFile}`));
+
+  const {width, height} = imgOld; // new image should have same w/h
+
+  const diff = new PNG({width, height});
+
+  const pxDiffCount = pixelmatch(imgOld.data, imgNew.data, diff.data, width, height, {threshold: 0.1});
+
+  const logMsg = `px diff count: ${pxDiffCount} for ${imgFile}`;
+  if (pxDiffCount > 10) {
+    fs.writeFileSync(`./${diffDir}/${imgFile}`, PNG.sync.write(diff));
+
+    const combined = await combineImage([`./${diffDir}/${imgFile}`, `./${oldDir}/${imgFile}`, `./${newDir}/${imgFile}`])
+    await combined.write(`./${diffDir}/combined-${imgFile}`);
+
+    console.warn(logMsg);
+  } else {
+    console.log(logMsg);
+  }
+
+}

src/test/visual-regression/fetch-all-images.js

@@ -0,0 +1,82 @@
+import SwaggerParser from "@apidevtools/swagger-parser";
+import fetch from 'node-fetch';
+import {createWriteStream} from 'node:fs';
+import {pipeline} from 'node:stream';
+import {promisify} from 'node:util'
+import pdf2img from "pdf-img-convert"
+import fs from "fs";
+import {PNG} from "pngjs";
+
+// const baseUrl = "https://energy-label-service-qa.london.cloudapps.digital";
+// const folderName = "old";
+
+// const baseUrl = "https://dev.energylabels.energysecurity.gov.uk";
+// const folderName = "new";
+
+const baseUrl = "http://localhost:8080";
+const folderName = "new";
+
+function wait(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+SwaggerParser.dereference(baseUrl+"/api/v1/energy-labels-openapi.json").then(async api => {
+
+  for (const [path, obj] of Object.entries(api.paths)) {
+
+    await wait(500);
+
+    const url = baseUrl + path;
+    const requestBodyObj = {};
+    for (const [name, p] of Object.entries(obj.post.requestBody.content['application/json'].schema.properties)) {
+      requestBodyObj[name] = p.example;
+    }
+
+    // Energy labels and fiches
+    if (requestBodyObj.outputFormat !== undefined) {
+      // generate both PNG and PDF versions.
+      requestBodyObj.outputFormat = 'PNG';
+      const pngOutputPath = `./${folderName}/${path.replaceAll('/', '_')}-PNG.png`;
+      await doRequest(url, requestBodyObj, pngOutputPath);
+      const nativePng = PNG.sync.read(fs.readFileSync(pngOutputPath));
+      const pngWidth = nativePng.width;
+      const pngHeight = nativePng.height;
+      console.log(`${path} PNG OK`);
+
+      await wait(200);
+      requestBodyObj.outputFormat = 'PDF';
+      const pdfOutputPath = `./${folderName}/${path.replaceAll('/', '_')}-PDF.pdf`
+      await doRequest(url, requestBodyObj, pdfOutputPath);
+
+      // convert PDF to PNG, so we can check the PDFs have also generated ok
+      const imageOut = await pdf2img.convert(pdfOutputPath, {width: pngWidth, height: pngHeight});
+      for (let i = 0; i < imageOut.length; i++) {
+        fs.writeFileSync(pdfOutputPath.replace(".pdf", ".png"), imageOut[i]);
+      }
+
+      console.log(`${path} PDF OK`);
+    }
+
+    await wait(200);
+    // Internet arrows
+    if (requestBodyObj.labelFormat !== undefined) {
+      requestBodyObj.labelFormat = 'PNG';
+      await doRequest(url, requestBodyObj, `./${folderName}/${path.replaceAll('/', '_')}.png`);
+      console.log(`${path} PNG OK`);
+    }
+
+  }
+})
+
+async function doRequest(url, requestBody, outputFilePath) {
+  const response = await fetch(url, {
+    method: 'post',
+    body: JSON.stringify(requestBody),
+    headers: {'Content-Type': 'application/json'}
+  });
+  const streamPipeline = await promisify(pipeline);
+  await streamPipeline(response.body, createWriteStream(outputFilePath));
+}
+
+
+