Only download url2green when asked for. (#4354)

* Only download url2green when asked for.

The URL2green file is becoming larger and larger. Skip downloading
it if you don't specifically ask for it.

.dockerignore

@@ -5,3 +5,4 @@
 !LICENSE
 !npm-shrinkwrap.json
 !docker
+!tools/postinstall.js

Dockerfile

@@ -11,7 +11,6 @@ COPY docker/webpagereplay/wpr_key.pem /webpagereplay/certs/
 COPY docker/webpagereplay/deterministic.js /webpagereplay/scripts/deterministic.js
 COPY docker/webpagereplay/LICENSE /webpagereplay/
 
-
 RUN sudo apt-get update && sudo apt-get install libnss3-tools python2 \
     net-tools \
     build-essential \
@@ -28,6 +27,7 @@ WORKDIR /usr/src/app
 
 COPY package.json /usr/src/app/
 COPY npm-shrinkwrap.json /usr/src/app/
+COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
 RUN npm install --production && npm cache clean --force
 COPY . /usr/src/app
 

Dockerfile-slim

@@ -21,6 +21,7 @@ RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 COPY . /usr/src/app
+COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
 RUN CHROMEDRIVER_SKIP_DOWNLOAD=true EGDEDRIVER_SKIP_DOWNLOAD=true npm install --production && npm cache clean --force && npm uninstall npm -g
 WORKDIR /usr/src/app
 COPY docker/scripts/start-slim.sh /start.sh

lib/plugins/sustainable/index.js

@@ -91,7 +91,15 @@ export default class SustainablePlugin extends SitespeedioPlugin {
       !this.sustainableOptions.useGreenWebHostingAPI
     ) {
       log.info('Reading green domain data from disk');
-      this.data = await loadJSON(greenDomainJSONpath);
+      try {
+        this.data = await loadJSON(greenDomainJSONpath);
+      } catch {
+        log.error('Green domain local file is missing on disk.');
+        log.error(
+          'Run `DOWNLOAD_URL2GREEN=true npm install sitespeed.io` when you install to install the file locally or use the online API using `--sustainable.useGreenWebHostingAPI true` to check if a domain is green hosted.'
+        );
+        this.data = {};
+      }
     }
   }
 

package.json

@@ -56,7 +56,8 @@
     "build:css": "npm run build:css-light && npm run build:css-dark",
     "build:css-light": "sass lib/plugins/html/src/sass/main-light.scss > lib/plugins/html/assets/css/index-light.css && cleancss -o lib/plugins/html/assets/css/index-light.min.css lib/plugins/html/assets/css/index-light.css && rm lib/plugins/html/assets/css/index-light.css ",
     "build:css-dark": "sass lib/plugins/html/src/sass/main-dark.scss > lib/plugins/html/assets/css/index-dark.css && cleancss -o lib/plugins/html/assets/css/index-dark.min.css lib/plugins/html/assets/css/index-dark.css && rm lib/plugins/html/assets/css/index-dark.css",
-    "generate:assets": "mkdir -p assets/$npm_package_version && cp -R lib/plugins/html/assets/ assets/$npm_package_version/"
+    "generate:assets": "mkdir -p assets/$npm_package_version && cp -R lib/plugins/html/assets/ assets/$npm_package_version/",
+    "postinstall": "node tools/postinstall.js"
   },
   "engines": {
     "node": ">=14.19.1"

tools/postinstall.js

@@ -0,0 +1,61 @@
+import { existsSync, writeFileSync } from 'node:fs';
+import { mkdirSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+import path from 'node:path';
+import https from 'node:https';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const DEFAULT_URL2GREEN_URL =
+  'https://raw.githubusercontent.com/sitespeedio/url2green/main/url2green.json.gz';
+
+const URL2GREEN_FILE_PATH = `${__dirname}/../lib/plugins/sustainable/data/url2green.json.gz`;
+
+const DOWNLOAD_URL2GREEN = process.env.DOWNLOAD_URL2GREEN || 'false';
+
+function downloadFile(url, destinationPath) {
+  return new Promise((resolve, reject) => {
+    https
+      .get(url, response => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`Failed to get '${url}' (${response.statusCode})`));
+          return;
+        }
+
+        const fileChunks = [];
+        response.on('data', chunk => fileChunks.push(chunk));
+        response.on('end', () => {
+          try {
+            const fileBuffer = Buffer.concat(fileChunks);
+            mkdirSync(path.dirname(destinationPath), { recursive: true });
+            writeFileSync(destinationPath, fileBuffer);
+            resolve();
+          } catch (error) {
+            reject(error);
+          }
+        });
+      })
+      .on('error', reject);
+  });
+}
+
+async function run() {
+  if (DOWNLOAD_URL2GREEN === 'true') {
+    if (existsSync(URL2GREEN_FILE_PATH)) {
+      console.log('URL2GREEN file already exists. Skipping download.');
+    } else {
+      console.log('URL2GREEN file is missing. Downloading...');
+      try {
+        await downloadFile(DEFAULT_URL2GREEN_URL, URL2GREEN_FILE_PATH);
+        console.log('URL2GREEN file downloaded successfully.');
+      } catch (error) {
+        console.error('Failed to download URL2GREEN file:', error.message);
+      }
+    }
+  } else {
+    console.log('Skipping URL2GREEN file download (DOWNLOAD_URL2GREEN=false).');
+  }
+}
+
+await run();