Demo Mode (#117)

* Adding Demo Mode to Fredy

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"d","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

index.js

@@ -1,12 +1,14 @@
 import fs from 'fs';
-import { config } from './lib/utils.js';
+import {config} from './lib/utils.js';
 import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
 import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
 import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
 import {track} from './lib/services/tracking/Tracker.js';
+import {handleDemoUser} from './lib/services/storage/userStorage.js';
+import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -17,35 +19,43 @@ const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
 const INTERVAL = config.interval * 60 * 1000;
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+if(config.demoMode){
+    console.info('Running in demo mode');
+    cleanupDemoAtMidnight();
+}
 /* eslint-enable no-console */
 const fetchedProvider = await Promise.all(
   provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
 );
 
+handleDemoUser();
+
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-    if (isDuringWorkingHoursOrNotSet) {
-      track();
-      config.lastRun = Date.now();
-      jobStorage
-        .getJobs()
-        .filter((job) => job.enabled)
-        .forEach((job) => {
-          job.provider
-            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-            .forEach(async (prov) => {
-              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-              pro.init(prov, job.blacklist);
-              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-              setLastJobExecution(job.id);
-            });
-        });
-    } else {
-      /* eslint-disable no-console */
-      console.debug('Working hours set. Skipping as outside of working hours.');
-      /* eslint-enable no-console */
-    }
+        if(!config.demoMode) {
+            if (isDuringWorkingHoursOrNotSet) {
+                track();
+                config.lastRun = Date.now();
+                jobStorage
+                    .getJobs()
+                    .filter((job) => job.enabled)
+                    .forEach((job) => {
+                        job.provider
+                            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+                            .forEach(async (prov) => {
+                                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+                                pro.init(prov, job.blacklist);
+                                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+                                setLastJobExecution(job.id);
+                            });
+                    });
+            } else {
+                /* eslint-disable no-console */
+                console.debug('Working hours set. Skipping as outside of working hours.');
+                /* eslint-enable no-console */
+            }
+        }
     return exec;
   })(),
   INTERVAL

lib/api/api.js

@@ -12,6 +12,7 @@ import restana from 'restana';
 import files from 'serve-static';
 import path from 'path';
 import { getDirName } from '../utils.js';
+import {demoRouter} from './routes/demoRouter.js';
 const service = restana();
 const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
@@ -30,6 +31,9 @@ service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
+
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);

lib/api/routes/demoRouter.js

@@ -0,0 +1,11 @@
+import restana from 'restana';
+import {config} from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, {demoMode: config.demoMode});
+  res.send();
+});
+
+export { demoRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,6 +1,7 @@
 import restana from 'restana';
 import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
 import fs from 'fs';
+import {handleDemoUser} from '../../services/storage/userStorage.js';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
 generalSettingsRouter.get('/', async (req, res) => {
@@ -10,9 +11,14 @@ generalSettingsRouter.get('/', async (req, res) => {
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
   try {
+    if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      return;
+    }
     const currentConfig = await readConfigFromStorage();
     fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
     await refreshConfig();
+    handleDemoUser();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));

lib/api/routes/jobRouter.js

@@ -6,6 +6,7 @@ import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
+import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -68,6 +69,11 @@ jobRouter.post('/', async (req, res) => {
     res.send(new Error(error));
     console.error(error);
   }
+  trackDemoJobCreated({
+    name,
+    provider,
+    adapter: notificationAdapter
+  });
   res.send();
 });
 jobRouter.delete('', async (req, res) => {

lib/api/routes/loginRoute.js

@@ -1,6 +1,8 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as hasher from '../../services/security/hash.js';
+import {config} from '../../utils.js';
+import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
 const service = restana();
 const loginRouter = service.newRouter();
 loginRouter.get('/user', async (req, res) => {
@@ -24,6 +26,11 @@ loginRouter.post('/', async (req, res) => {
     return;
   }
   if (user.password === hasher.hash(password)) {
+
+    if(config.demoMode){
+      trackDemoAccessed();
+    }
+
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
     res.send(200);

lib/api/routes/userRoute.js

@@ -1,6 +1,7 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as jobStorage from '../../services/storage/jobStorage.js';
+import {config} from '../../utils.js';
 const service = restana();
 const userRouter = service.newRouter();
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -20,6 +21,11 @@ userRouter.get('/:userId', async (req, res) => {
   res.send();
 });
 userRouter.delete('/', async (req, res) => {
+  if(config.demoMode){
+    res.send(new Error('In demo mode, it is not allowed to remove user.'));
+    return;
+  }
+
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
@@ -36,6 +42,12 @@ userRouter.delete('/', async (req, res) => {
   res.send();
 });
 userRouter.post('/', async (req, res) => {
+
+  if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+      return;
+  }
+
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
     res.send(new Error('Passwords does not match'));

lib/services/demoCleanup.js

@@ -0,0 +1,29 @@
+import { setInterval } from 'node:timers';
+import {removeJobsByUserName} from './storage/jobStorage.js';
+import {config} from '../utils.js';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+    const now = new Date();
+    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+        - now.getUTCMinutes() * 60 * 1000
+        - now.getUTCSeconds() * 1000
+        - now.getUTCMilliseconds();
+
+    setTimeout(() => {
+        cleanup();
+
+        setInterval(() => {
+            cleanup();
+        }, 24 * 60 * 60 * 1000);
+
+    }, millisUntilMidnightUTC);
+}
+
+function cleanup(){
+    if(config.demoMode){
+        removeJobsByUserName('demo');
+    }
+}

lib/services/storage/jobStorage.js

@@ -77,6 +77,17 @@ export const removeJobsByUserId = (userId) => {
     .value();
   db.write();
 };
+export const removeJobsByUserName = (userName) => {
+  db.chain
+      .get('jobs')
+      .filter((job) => job.username === userName)
+      .forEach((job) => listingStorage.removeListings(job.id));
+  db.chain
+      .get('jobs')
+      .remove((job) => job.username === userName)
+      .value();
+  db.write();
+};
 export const getJobs = () => {
   return db.chain
     .get('jobs')

lib/services/storage/userStorage.js

@@ -1,5 +1,5 @@
 import { JSONFileSync } from 'lowdb/node';
-import { getDirName } from '../../utils.js';
+import {config, getDirName} from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import * as jobStorage from './jobStorage.js';
@@ -16,6 +16,13 @@ const defaultData = {
             password: hasher.hash('admin'),
             isAdmin: true,
         },
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'demo',
+            password: hasher.hash('demo'),
+            isAdmin: true,
+        },
     ],
 };
 
@@ -84,3 +91,29 @@ export const removeUser = (userId) => {
     .value();
   db.write();
 };
+
+export const handleDemoUser = () => {
+    if(!config.demoMode){
+        const user = db.chain.get('user').value();
+        db.chain.get('user').value();
+        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+        db.write();
+    }else {
+        const demoUser = db.chain
+            .get('user')
+            .filter((u) => u.username === 'demo')
+            .value();
+        if (demoUser == null || demoUser.length === 0) {
+            db.chain.get('user')
+                .value()
+                .push({
+                    id: nanoid(),
+                    username: 'demo',
+                    password: hasher.hash('demo'),
+                    isAdmin: true,
+                });
+            db.write();
+        }
+    }
+};
+

lib/services/tracking/Tracker.js

@@ -1,20 +1,16 @@
 import Mixpanel from 'mixpanel';
 import {getJobs} from '../storage/jobStorage.js';
 
-import {config} from '../../utils.js';
+import {config, inDevMode} from '../../utils.js';
+
+const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
 
 export const track = function () {
     //only send tracking information if the user allowed to do so.
-    if (config.analyticsEnabled) {
-
-        const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+    if (config.analyticsEnabled && !inDevMode()) {
 
         const activeProvider = new Set();
         const activeAdapter = new Set();
-        const platform = process.platform;
-        const arch = process.arch;
-        const language = process.env.LANG || 'en';
-        const nodeVersion = process.version || 'N/A';
 
         const jobs = getJobs();
 
@@ -28,15 +24,45 @@ export const track = function () {
                 });
             });
 
-            mixpanelTracker.track('fredy_tracking', {
+            mixpanelTracker.track('fredy_tracking', enrichTrackingObject({
                 adapter: Array.from(activeAdapter),
                 provider: Array.from(activeProvider),
-                isDemo: config.demoMode,
-                platform,
-                arch,
-                nodeVersion,
-                language
-            });
+            }));
         }
     }
-};
+};
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoJobCreated(jobData) {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+    }
+}
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoAccessed() {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
+    }
+}
+
+
+function enrichTrackingObject(trackingObject) {
+    const platform = process.platform;
+    const arch = process.arch;
+    const language = process.env.LANG || 'en';
+    const nodeVersion = process.version || 'N/A';
+
+    return {
+        ...trackingObject,
+        isDemo: config.demoMode,
+        platform,
+        arch,
+        nodeVersion,
+        language
+    };
+}