Upgrade report data pipelines

definitions/output/reports/dynamic_publisher.js

@@ -0,0 +1,16 @@
+const configs = new reports.HTTPArchiveReports()
+const params = {
+  date: constants.currentMonth,
+  rankFilter: constants.devRankFilter
+}
+
+const metrics = configs.listMetrics()
+metrics.forEach(metric => {
+  metric.SQL.forEach(sql => {
+    publish(sql.type, {
+      type: 'table',
+      schema: 'reports',
+      tags: ['crawl_reports']
+    }).query(ctx => constants.fillTemplate(sql.query, params))
+  })
+})

includes/constants.js

@@ -1,7 +1,9 @@
 const today = (dataform.projectConfig.vars.today ? dataform.projectConfig.vars.today : new Date().toISOString()).substring(0, 10)
 const currentMonth = today.substring(0, 8) + '01'
-const fnDateUnderscored = (dateStr) => dateStr.replaceAll('-', '_')
-const fnPastMonth = (monthISOstring) => {
+function fnDateUnderscored (dateStr) {
+  return dateStr.replaceAll('-', '_')
+}
+function fnPastMonth (monthISOstring) {
   const monthDate = new Date(monthISOstring)
   monthDate.setMonth(monthDate.getMonth() - 1)
   return monthDate.toISOString().substring(0, 10)
@@ -17,6 +19,12 @@ const [
       'AND rank <= 10000'
     ]
   : ['', '']
+function fillTemplate (template, params) {
+  return template.replace(/{{(.*?)}}/g, (match, key) => {
+    const trimmedKey = key.trim()
+    return trimmedKey in params ? params[trimmedKey] : match
+  })
+}
 
 module.exports = {
   today,
@@ -26,5 +34,6 @@ module.exports = {
   clients,
   booleans,
   devTABLESAMPLE,
-  devRankFilter
+  devRankFilter,
+  fillTemplate
 }

includes/reports.js

@@ -0,0 +1,62 @@
+const { config } = require('./reports_config')
+
+class HTTPArchiveReports {
+  constructor () {
+    this.config = config
+  }
+
+  listReports () {
+    const reportIds = this.config._reports
+
+    const reports = reportIds.map(reportId => {
+      const report = this.getReport(reportId)
+      return report
+    })
+
+    console.log('reports', reports)
+
+    return reports
+  }
+
+  getReport (reportId) {
+    const report = this.config[reportId]
+    return {
+      id: reportId,
+      ...report
+    }
+  }
+
+  listMetrics (reportId) {
+    if (reportId === undefined) {
+      const metrics = Object.keys(this.config._metrics).map(metricId => {
+        const metric = this.getMetric(metricId)
+        return metric
+      }).filter(metric => metric.SQL)
+
+      return metrics
+    } else {
+      const report = this.getReport(reportId)
+      const metricIds = report.metrics
+
+      const metrics = metricIds.map(metricId => {
+        const metric = this.getMetric(metricId)
+        return metric
+      }).filter(metric => metric.SQL)
+
+      return metrics
+    }
+  }
+
+  getMetric (metricId) {
+    const metric = this.config._metrics[metricId]
+
+    return {
+      id: metricId,
+      ...metric
+    }
+  }
+}
+
+module.exports = {
+  HTTPArchiveReports
+}