ci: install xvfb on the machine and start it with pm2

.dockerignore

@@ -1,6 +1,8 @@
 __mocks__
 .husky
 **/build
-**/docker
+**/docker/electron-builder
 **/coverage
 **/node_modules
+docker
+!docker/guardoni/guardoni.config.json

.github/workflows/guardoni_pull_request.yml

@@ -10,8 +10,8 @@ on:
       - fix/**
       - refactor/**
     paths:
-      - "packages/shared/**"
-      - "platforms/**"
+      - 'packages/shared/**'
+      - 'platforms/**'
 
 jobs:
   pull_request:
@@ -34,7 +34,7 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
-          cache: "yarn"
+          cache: 'yarn'
           cache-dependency-path: yarn.lock
 
       - name: Install modules
@@ -69,9 +69,6 @@ jobs:
   guardoni_build:
     name: Guardoni build
     runs-on: ${{ matrix.config.os }}
-    # defaults:
-    #   run:
-    #     shell: bash --noprofile --norc -exo pipefail {0}
     strategy:
       max-parallel: 1
       matrix:
@@ -126,21 +123,28 @@ jobs:
             platforms/guardoni/dist
             platforms/ycai/studio/build
 
+      - name: Install Xvfb
+        run: |
+          sudo apt install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 xdotool
+
       - name: Install dependencies
         run: yarn
 
       - name: Start PM2
-        run: |
-          yarn pm2 start --env test ./platforms/ecosystem.config.js
-          sleep 5
+        run: yarn pm2 start --env test ./platforms/ecosystem.config.js
 
       - name: Build Guardoni for ${{ matrix.config.os }}
         working-directory: ./platforms/guardoni
         run: ./scripts/cli-build.mjs
 
-      - name: Test YT cli
+      - name: Test CLI - YT Home
         working-directory: ./platforms/guardoni
-        run: ./scripts/cli-yt-test.mjs
+        env:
+          DISPLAY: ':99.0'
+        run: xvfb-run --auto-servernum  ./scripts/cli-yt-test-home.mjs
+
+      - name: Test CLI - YT Videos
+        run: yarn guardoni cli-yt-test-videos
 
       - name: Stop PM2
         run: yarn pm2 stop all

docker/guardoni/Dockerfile

@@ -0,0 +1,31 @@
+FROM node:16-slim
+
+RUN mkdir /guardoni
+WORKDIR /guardoni
+RUN apt-get update
+RUN apt-get install -y python3 build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev
+
+ADD ./ ./
+
+RUN yarn install
+RUN yarn tk:ext build
+RUN yarn yt:ext build
+RUN yarn guardoni build:cli
+RUN yarn guardoni pkg
+WORKDIR /guardoni/platforms/guardoni/dist/
+RUN mv $(ls -1 guardoni-cli*-linux) guardoni-cli
+
+FROM selenium/standalone-chrome:102.0
+WORKDIR /guardoni
+
+COPY --from=0 /guardoni/platforms/guardoni/dist/guardoni-cli guardoni-cli
+COPY --from=0 "/guardoni/node_modules/@tktrex/extension/build" tktrex
+COPY --from=0 "/guardoni/node_modules/@yttrex/extension/build" yttrex
+COPY --from=0 /guardoni/docker/guardoni/guardoni.config.json .
+
+USER root
+RUN chmod -R 777 /guardoni/yttrex
+RUN chmod -R 777 /guardoni/tktrex
+USER seluser
+
+ENTRYPOINT [ "/guardoni/guardoni-cli" ]

docker/guardoni/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
+cd "$SCRIPT_DIR/../../"
+docker build -f "docker/guardoni/Dockerfile" -t guardoni .

docker/guardoni/guardoni.config.json

@@ -0,0 +1,17 @@
+{
+  "profileName": "dockerized-guardoni",
+  "headless": true,
+  "verbose": false,
+  "loadFor": 3000,
+  "yt": {
+    "name": "youtube",
+    "backend": "https://youtube.tracking.exposed/api",
+    "extensionDir": "/guardoni/yttrex"
+  },
+  "tk": {
+    "name": "tiktok",
+    "backend": "https://tiktok.tracking.exposed/api",
+    "extensionDir": "/guardoni/tktrex"
+  },
+  "chromePath": "/usr/bin/google-chrome"
+}

docs/package.json

@@ -27,7 +27,7 @@
     "@tsconfig/docusaurus": "^1.0.4",
     "docusaurus-plugin-openapi": "^0.5.0",
     "docusaurus-preset-openapi": "^0.5.0",
-    "typescript": "^4.7.2"
+    "typescript": "^4.7.4"
   },
   "browserslist": {
     "production": [

package.json

@@ -13,7 +13,6 @@
     "packages/*",
     "platforms/guardoni",
     "platforms/ycai/studio",
-    "platforms/makhno/*",
     "platforms/yttrex/*",
     "platforms/tktrex/docs",
     "platforms/tktrex/extension",
@@ -26,7 +25,7 @@
     "clean": "yarn workspaces foreach -v run clean",
     "lint": "yarn workspaces foreach -v run lint",
     "type-check": "yarn workspaces foreach run tsc --noEmit",
-    "build": "yarn workspaces foreach run build",
+    "build": "scripts/build.sh",
     "test": "jest",
     "docs": "yarn workspace @trex/docs",
     "taboule": "yarn workspace @trex/taboule",
@@ -40,7 +39,6 @@
     "yt:shared": "yarn workspace @yttrex/shared",
     "yt:backend": "yarn workspace @yttrex/backend",
     "yt:ext": "yarn workspace @yttrex/extension",
-    "makhno": "yarn workspace @makhno/backend",
     "ycai": "yarn workspace @trex/ycai",
     "lint-staged": "lint-staged",
     "postinstall": "husky install",
@@ -57,16 +55,16 @@
     "@release-it/conventional-changelog": "^4.3.0",
     "@types/node": "^16.11.36",
     "@types/prettier": "^2.6.3",
-    "eslint": "^8.16.0",
+    "eslint": "^8.19.0",
+    "eslint-plugin-n": "^15.2.4",
     "husky": "^7.0.4",
     "jest": "^27.5.1",
-    "jest-dev-server": "^6.0.3",
     "lint-staged": "^12.3.3",
-    "prettier": "^2.6.2",
+    "prettier": "^2.7.1",
     "release-it": "^14.14.3",
     "release-it-yarn-workspaces": "^2.0.1",
     "ts-jest": "^27.1.5",
-    "typescript": "^4.7.2",
+    "typescript": "^4.7.4",
     "webpack": "^5.73.0",
     "zx": "^6.2.5"
   },

packages/shared/package.json

@@ -24,44 +24,44 @@
     "io-ts-types": "^0.5.16",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
-    "react-i18next": "^11.15.3",
+    "react-i18next": "^11.18.0",
     "ts-endpoint": "^2.0.0",
     "ts-endpoint-express": "^2.0.0",
     "ts-io-error": "^2.0.0",
     "typelevel-ts": "^0.4.0",
-    "typescript": "^4.7.2"
+    "typescript": "^4.7.4"
   },
   "devDependencies": {
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.7",
     "@types/chrome": "^0.0.188",
     "@types/debug": "^4.1.7",
     "@types/dotenv-webpack": "^7.0.3",
     "@types/node": "^16.11.36",
-    "@types/react": "^17.0.45",
+    "@types/react": "^17.0.47",
     "@types/react-dom": "^17.0.17",
     "@types/webpack-bundle-analyzer": "^4.4.1",
-    "@typescript-eslint/eslint-plugin": "^5.27.0",
-    "@typescript-eslint/parser": "^5.27.0",
+    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/parser": "^5.30.5",
     "copy-webpack-plugin": "^11.0.0",
     "dotenv-webpack": "^7.1.0",
-    "eslint": "^8.16.0",
+    "eslint": "^8.19.0",
     "eslint-config-prettier": "^8.5.0",
-    "eslint-config-standard-with-typescript": "^21.0.1",
+    "eslint-config-standard-with-typescript": "^22.0.0",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^5.2.0",
-    "eslint-plugin-react": "^7.30.0",
+    "eslint-plugin-promise": "^6.0.0",
+    "eslint-plugin-react": "^7.30.1",
     "fast-check": "^2.25.0",
     "fast-check-io-ts": "^0.5.0",
-    "filemanager-webpack-plugin": "^6.1.7",
+    "filemanager-webpack-plugin": "^7.0.0",
     "html-webpack-plugin": "^5.5.0",
     "jest": "^27.5.1",
     "mini-css-extract-plugin": "^2.6.1",
     "node-loader": "^2.0.0",
     "node-sass": "^7.0.1",
-    "prettier": "^2.6.2",
+    "prettier": "^2.7.1",
     "react-refresh": "^0.12.0",
-    "react-refresh-typescript": "^2.0.5",
+    "react-refresh-typescript": "^2.0.7",
     "sass-loader": "^12.6.0",
     "style-loader": "^3.3.1",
     "tsconfig-paths-webpack-plugin": "^3.5.2",

packages/shared/src/backend/utils/endpoint.ts

@@ -37,7 +37,7 @@ const toError = (e: unknown): IOError => {
   };
 };
 
-declare type UndefinedOrRuntime<N> = N extends RecordCodec<any, any>
+export declare type UndefinedOrRuntime<N> = N extends RecordCodec<any, any>
   ? {
       [k in keyof N['props']]: runtimeType<N['props'][k]>;
     }

packages/shared/src/backend/utils/routeHandlerMiddleware.ts

@@ -0,0 +1,131 @@
+import express from 'express';
+import _ from 'lodash';
+import { APIError } from '../../errors/APIError';
+import { GetLogger } from '../../logger';
+
+const logger = GetLogger('route-handler');
+
+const logAPICount = { requests: {}, responses: {}, errors: {} };
+
+function loginc(kind: string, fname: string): void {
+  (logAPICount as any)[kind][fname] = (logAPICount as any)[kind][fname]
+    ? (logAPICount as any)[kind][fname]++
+    : 1;
+}
+
+type RouteHandlerMiddleware<R extends Record<string, express.RequestHandler>> =
+  (
+    fname: keyof R
+  ) => (req: express.Request, res: express.Response) => Promise<void>;
+
+export const routeHandleMiddleware = <
+  R extends Record<string, express.RequestHandler>
+>(
+  apiList: R
+): RouteHandlerMiddleware<R> => {
+  setInterval(() => {
+    let print = false;
+    _.each(_.keys(logAPICount), function (k) {
+      if (!_.keys((logAPICount as any)[k]).length)
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete (logAPICount as any)[k];
+      else print = true;
+    });
+    if (print) logger.debug('%j', logAPICount);
+    logAPICount.responses = {};
+    logAPICount.errors = {};
+    logAPICount.requests = {};
+  }, 60 * 1000);
+
+  return (fn) => async (req, res) => {
+    const fname = fn as any;
+    try {
+      loginc('requests', fname);
+      const funct = apiList[fname];
+      const httpresult = await (funct as any)(req, res);
+
+      if (!httpresult) {
+        logger.debug("API (%s) didn't return anything!?", fname);
+        loginc('errors', fname);
+        res.send('Fatal error: Invalid output');
+        res.status(501);
+        return;
+      }
+
+      if (httpresult.json?.error) {
+        const statusCode = httpresult.json.status ?? 500;
+        logger.debug('API (%s) failure, returning %d', fname, statusCode);
+        loginc('errors', fname);
+        res.status(statusCode);
+        res.json(httpresult.json);
+        return;
+      }
+
+      if (httpresult.headers)
+        _.each(httpresult.headers, function (value, key) {
+          logger.debug('Setting header %s: %s', key, value);
+          res.setHeader(key, value);
+        });
+
+      if (httpresult.json) {
+        // logger("API (%s) success, returning %d bytes JSON", fname, _.size(JSON.stringify(httpresult.json)));
+        loginc('responses', fname);
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.json(httpresult.json);
+      } else if (httpresult.text) {
+        // logger("API (%s) success, returning text (size %d)", fname, _.size(httpresult.text));
+        loginc('responses', fname);
+        res.send(httpresult.text);
+      } else if (httpresult.status) {
+        // logger("Returning empty status %d from API (%s)", httpresult.status, fname);
+        loginc('responses', fname);
+        res.status(httpresult.status);
+      } else {
+        logger.debug(
+          'Undetermined failure in API (%s) →  %j',
+          fname,
+          httpresult
+        );
+        loginc('errors', fname);
+        res.status(502);
+        res.send('Error?');
+      }
+    } catch (error) {
+      logger.error('Route handler (%s) error: %O', fname, error);
+
+      // todo: this should be 500
+      res.status(502);
+
+      if (error instanceof APIError) {
+        logger.error(
+          'APIError - %s: (%s) %s %s',
+          error.name,
+          error.message,
+          error.stack
+        );
+        res.status(error.status);
+        res.send({
+          name: error.type,
+          message: error.message,
+          details: error.details,
+        });
+      } else if (error instanceof Error) {
+        logger.error('Error - %s: %s', error.name, error.message);
+
+        res.send('Software error: ' + error.message);
+        logger.error(
+          'Error in HTTP handler API(%s): %s %s',
+          fname,
+          error.message,
+          error.stack
+        );
+      } else {
+        res.status(502);
+        res.send('Software error: ' + (error as any).message);
+        loginc('errors', fname);
+        logger.debug('Error in HTTP handler API(%s): %o', fname, error);
+      }
+    }
+    res.end();
+  };
+};