feat: read scrape instructions from app root dir (#10)

* chore: add more IPs to user-impersonation whitelist to allow debugging from outside docker too * feat: read scrape instructions from push-analytics.json at app url
dhis2 · Mar 6, 2024 · 3bf208a · 3bf208a
1 parent 33853bb
commit 3bf208a
Show file tree

Hide file tree

Showing 11 changed files with 104 additions and 295 deletions.
diff --git a/docker/dhis.conf b/docker/dhis.conf
@@ -4,6 +4,6 @@ connection.url = jdbc:postgresql://db/dhis
 connection.username = dhis
 connection.password = dhis
 switch_user_feature.enabled = true
-switch_user_allow_listed_ips = 172.26.0.10
+switch_user_allow_listed_ips = 172.26.0.1,172.26.0.10,192.168.65.1
 
 tracker.import.preheat.cache.enabled = off
diff --git a/src/dummy-instructions/data-visualizer-app.json b/src/dummy-instructions/data-visualizer-app.json
diff --git a/src/dummy-instructions/event-charts-app.json b/src/dummy-instructions/event-charts-app.json
diff --git a/src/dummy-instructions/event-reports-app.json b/src/dummy-instructions/event-reports-app.json
diff --git a/src/dummy-instructions/line-listing-app.json b/src/dummy-instructions/line-listing-app.json
diff --git a/src/dummy-instructions/maps-app.json b/src/dummy-instructions/maps-app.json
diff --git a/src/types/ConverterCluster.ts b/src/types/ConverterCluster.ts
@@ -1,5 +1,6 @@
 import { Browser } from 'puppeteer'
 import type { DashboardItem } from '.'
+import { Authenticator } from '../worker/Authenticator'
 
 export type OnConversionCompleteFn = (html: string) => void
 
@@ -54,6 +55,6 @@ export type ConverterResult = {
 
 export interface Converter {
     convert: (queueItem: QueueItem) => Promise<ConverterResult>
-    init?: (browser: Browser) => Promise<void>
+    init?: (browser: Browser, authenticator: Authenticator) => Promise<void>
     takeErrorScreenShot?: (queueItem: QueueItem) => Promise<void>
 }
diff --git a/src/worker/AppScraper.ts b/src/worker/AppScraper.ts
@@ -21,6 +21,7 @@ import {
     waitForFileToDownload,
 } from '../utils'
 import { ScrapeConfigCache } from './ScrapeConfigCache'
+import { Authenticator } from './Authenticator'
 
 const DONWLOAD_PAGE_URL_PATTERN =
     /api\/analytics\/enrollments|events\/query\/[a-zA-Z0-9]{11}\.html\+css/
@@ -30,20 +31,25 @@ export class AppScraper implements Converter {
     #browser: Browser | null
     #page: Page | null
     #cdpSession: CDPSession | null
-    #configCache: ScrapeConfigCache
+    #configCache: ScrapeConfigCache | null
+    #authenticator: Authenticator | null
 
     constructor(baseUrl: string) {
         this.baseUrl = baseUrl
         this.#page = null
         this.#browser = null
         this.#cdpSession = null
-        this.#configCache = new ScrapeConfigCache(baseUrl)
+        this.#authenticator = null
+        this.#configCache = null
     }
 
-    async init(browser: Browser) {
+    async init(browser: Browser, authenticator: Authenticator) {
         this.#browser = browser
         this.#page = await browser.newPage()
         this.#cdpSession = await this.#page.target().createCDPSession()
+        this.#authenticator = authenticator
+        this.#configCache = new ScrapeConfigCache(this.baseUrl, authenticator)
+
         await this.#cdpSession.send('Browser.setDownloadBehavior', {
             behavior: 'allow',
             downloadPath,
@@ -66,6 +72,14 @@ export class AppScraper implements Converter {
         }
     }
 
+    get configCache() {
+        if (!this.#configCache) {
+            throw new Error('Config Cache has not been initialized')
+        } else {
+            return this.#configCache
+        }
+    }
+
     public async convert(queueItem: QueueItem): Promise<ConverterResult> {
         const visualization = getDashboardItemVisualization(
             queueItem.dashboardItem
@@ -83,7 +97,7 @@ export class AppScraper implements Converter {
 
         const timer = createTimer()
         await this.page.bringToFront()
-        const config = await this.#configCache.getScrapeConfig(
+        const config = await this.configCache.getScrapeConfig(
             queueItem.dashboardItem
         )