Initial commit

Created from https://vercel.com/new

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [lissy93]

.github/screenshots/README.md

@@ -0,0 +1 @@
+![Screenshot](https://raw.githubusercontent.com/Lissy93/web-check/HEAD/.github/screenshots/web-check-screenshot3.png)

.gitignore

@@ -0,0 +1,60 @@
+# ------------------------
+#   ENVIRONMENT SETTINGS
+# ------------------------
+.env
+
+# ------------------------
+#       PRODUCTION
+# ------------------------
+/build/
+
+# ------------------------
+#         DEPLOYMENT
+# ------------------------
+.vercel/
+.netlify/
+.webpack/
+.serverless/
+
+# ------------------------
+#         DEPENDENCIES
+# ------------------------
+node_modules/
+.yarn/cache/
+.yarn/unplugged/
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnpm/
+.pnp.*
+
+# ------------------------
+#            LOGS
+# ------------------------
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# ------------------------
+#          TESTING
+# ------------------------
+coverage/
+.nyc_output/
+
+# ------------------------
+#       OS SPECIFIC
+# ------------------------
+.DS_Store
+Thumbs.db
+
+# ------------------------
+#         EDITORS
+# ------------------------
+.idea/
+.vscode/
+*.swp
+*.swo
+

Dockerfile

@@ -0,0 +1,23 @@
+FROM node:16-buster-slim
+
+RUN apt-get update && \
+    apt-get install -y chromium traceroute && \
+    chmod 755 /usr/bin/chromium && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY package.json yarn.lock ./
+
+RUN yarn install && \
+    rm -rf /app/node_modules/.cache
+
+COPY . .
+
+RUN yarn build
+
+EXPOSE ${PORT:-3000}
+
+ENV CHROME_PATH='/usr/bin/chromium'
+
+CMD ["yarn", "serve"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Alicia Sykes <https://github.com/lissy93>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

api/_common/aws-webpack.config.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const nodeExternals = require('webpack-node-externals');
+
+module.exports = {
+    target: 'node',
+    mode: 'production',
+    entry: {
+        'carbon': './api/carbon.js',
+        'cookies': './api/cookies.js',
+        'dns-server': './api/dns-server.js',
+        'dns': './api/dns.js',
+        'dnssec': './api/dnssec.js',
+        'features': './api/features.js',
+        'get-ip': './api/get-ip.js',
+        'headers': './api/headers.js',
+        'hsts': './api/hsts.js',
+        'linked-pages': './api/linked-pages.js',
+        'mail-config': './api/mail-config.js',
+        'ports': './api/ports.js',
+        'quality': './api/quality.js',
+        'redirects': './api/redirects.js',
+        'robots-txt': './api/robots-txt.js',
+        'screenshot': './api/screenshot.js',
+        'security-txt': './api/security-txt.js',
+        'sitemap': './api/sitemap.js',
+        'social-tags': './api/social-tags.js',
+        'ssl': './api/ssl.js',
+        'status': './api/status.js',
+        'tech-stack': './api/tech-stack.js',
+        'trace-route': './api/trace-route.js',
+        'txt-records': './api/txt-records.js',
+        'whois': './api/whois.js',
+    },
+    externals: [nodeExternals()],
+    output: {
+        filename: '[name].js',
+        path: path.resolve(__dirname, '.webpack'),
+        libraryTarget: 'commonjs2'
+    },
+    module: {
+        rules: [
+            {
+                test: /\.js$/,
+                use: {
+                    loader: 'babel-loader'
+                },
+                exclude: /node_modules/,
+            }
+        ]
+    }
+};

api/_common/middleware.js

@@ -0,0 +1,82 @@
+const normalizeUrl = (url) => {
+  return url.startsWith('http') ? url : `https://${url}`;
+};
+
+const headers = {
+  'Access-Control-Allow-Origin': process.env.API_CORS_ORIGIN || '*',
+  'Access-Control-Allow-Credentials': true,
+  'Content-Type': 'application/json;charset=UTF-8',
+};
+
+const commonMiddleware = (handler) => {
+  // Vercel
+  const vercelHandler = async (request, response) => {
+    const queryParams = request.query || {};
+    const rawUrl = queryParams.url;
+
+    if (!rawUrl) {
+      return response.status(500).json({ error: 'No URL specified' });
+    }
+
+    const url = normalizeUrl(rawUrl);
+
+    try {
+      const handlerResponse = await handler(url, request);
+      if (handlerResponse.body && handlerResponse.statusCode) {
+        response.status(handlerResponse.statusCode).json(handlerResponse.body);
+      } else {
+        response.status(200).json(
+          typeof handlerResponse === 'object' ? handlerResponse : JSON.parse(handlerResponse)
+        );
+      }
+    } catch (error) {
+      response.status(500).json({ error: error.message });
+    }
+  };
+
+  // Netlify
+  const netlifyHandler = async (event, context, callback) => {
+    const queryParams = event.queryStringParameters || event.query || {};
+    const rawUrl = queryParams.url;
+
+    if (!rawUrl) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: 'No URL specified' }),
+        headers,
+      });
+      return;
+    }
+
+    const url = normalizeUrl(rawUrl);
+
+    try {
+      const handlerResponse = await handler(url, event, context);
+      if (handlerResponse.body && handlerResponse.statusCode) {
+        callback(null, handlerResponse);
+      } else {
+        callback(null, {
+          statusCode: 200,
+          body: typeof handlerResponse === 'object' ? JSON.stringify(handlerResponse) : handlerResponse,
+          headers,
+        });
+      }
+    } catch (error) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: error.message }),
+        headers,
+      });
+    }
+  };
+
+  // The format of the handlers varies between platforms
+  // E.g. Netlify + AWS expect Lambda functions, but Vercel or Node needs standard handler
+  const platformEnv = (process.env.PLATFORM || '').toUpperCase(); // Has user set platform manually?
+
+  const nativeMode = (['VERCEL', 'NODE'].includes(platformEnv) || process.env.VERCEL || process.env.WC_SERVER);
+
+  return nativeMode ? vercelHandler : netlifyHandler;
+};
+
+module.exports = commonMiddleware;

api/archives.js

@@ -0,0 +1,84 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const convertTimestampToDate = (timestamp) => {
+  const [year, month, day, hour, minute, second] = [
+    timestamp.slice(0, 4),
+    timestamp.slice(4, 6) - 1,
+    timestamp.slice(6, 8),
+    timestamp.slice(8, 10),
+    timestamp.slice(10, 12),
+    timestamp.slice(12, 14),
+  ].map(num => parseInt(num, 10));
+
+  return new Date(year, month, day, hour, minute, second);
+}
+
+const countPageChanges = (results) => {
+  let prevDigest = null;
+  return results.reduce((acc, curr) => {
+    if (curr[2] !== prevDigest) {
+      prevDigest = curr[2];
+      return acc + 1;
+    }
+    return acc;
+  }, -1);
+}
+
+const getAveragePageSize = (scans) => {
+    const totalSize = scans.map(scan => parseInt(scan[3], 10)).reduce((sum, size) => sum + size, 0);
+    return Math.round(totalSize / scans.length);
+};
+
+const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => {
+  const formatToTwoDecimal = num => parseFloat(num.toFixed(2));
+
+  const dayFactor = (lastScan - firstScan) / (1000 * 60 * 60 * 24);  
+  const daysBetweenScans = formatToTwoDecimal(dayFactor / totalScans);
+  const daysBetweenChanges = formatToTwoDecimal(dayFactor / changeCount);
+  const scansPerDay = formatToTwoDecimal((totalScans - 1) / dayFactor);
+  const changesPerDay = formatToTwoDecimal(changeCount / dayFactor);
+  return {
+    daysBetweenScans,
+    daysBetweenChanges,
+    scansPerDay,
+    changesPerDay,
+  };
+};
+
+const getWaybackData = async (url) => {
+  const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`;
+
+  try {
+    const { data } = await axios.get(cdxUrl);
+
+    // Check there's data
+    if (!data || !Array.isArray(data) || data.length <= 1) {
+      return { skipped: 'Site has never before been archived via the Wayback Machine' };
+    }
+
+    // Remove the header row
+    data.shift();
+
+    // Process and return the results
+    const firstScan = convertTimestampToDate(data[0][0]);
+    const lastScan = convertTimestampToDate(data[data.length - 1][0]);
+    const totalScans = data.length;
+    const changeCount = countPageChanges(data);
+    return {
+      firstScan,
+      lastScan,
+      totalScans,
+      changeCount,
+      averagePageSize: getAveragePageSize(data),
+      scanFrequency: getScanFrequency(firstScan, lastScan, totalScans, changeCount),
+      scans: data,
+      scanUrl: url,
+    };
+  } catch (err) {
+    return { error: `Error fetching Wayback data: ${err.message}` };
+  }
+};
+
+module.exports = middleware(getWaybackData);
+module.exports.handler = middleware(getWaybackData);