simplify tags

HTTPArchive · Dec 9, 2024 · f2b56f0 · f2b56f0
1 parent ade5867
commit f2b56f0
Show file tree

Hide file tree

Showing 15 changed files with 58 additions and 58 deletions.
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ Tag: `crawl_complete`
 
 ### Core Web Vitals Technology Report
 
-Tag: `cwv_tech_report`
+Tag: `crux_ready`
 
 - httparchive.core_web_vitals.technologies
 
@@ -26,7 +26,7 @@ Consumers:
 
 ### Blink Features Report
 
-Tag: `blink_features_report`
+Tag: `crawl_complete`
 
 - httparchive.blink_features.features
 - httparchive.blink_features.usage
@@ -39,11 +39,11 @@ Consumers:
 
 1. [crawl-complete](https://console.cloud.google.com/cloudpubsub/subscription/detail/dataformTrigger?authuser=7&project=httparchive) PubSub subscription
 
-    Tags: ["crawl_complete", "blink_features_report", "crawl_results_legacy"]
+    Tags: ["crawl_complete"]
 
 2. [bq-poller-cwv-tech-report](https://console.cloud.google.com/cloudscheduler/jobs/edit/us-east4/bq-poller-cwv-tech-report?authuser=7&project=httparchive) Scheduler
 
-    Tags: ["cwv_tech_report"]
+    Tags: ["crux_ready"]
 
 ### Triggering workflows
 

diff --git a/definitions/output/blink_features/features.js b/definitions/output/blink_features/features.js
@@ -6,7 +6,7 @@ publish('features', {
     partitionBy: 'yyyymmdd',
     clusterBy: ['client', 'rank']
   },
-  tags: ['blink_features_report']
+  tags: ['crawl_complete']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE yyyymmdd = DATE '${constants.currentMonth}';

diff --git a/definitions/output/blink_features/usage.js b/definitions/output/blink_features/usage.js
@@ -2,7 +2,7 @@ publish('usage', {
   schema: 'blink_features',
   type: 'incremental',
   protected: true,
-  tags: ['blink_features_report']
+  tags: ['crawl_complete']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE yyyymmdd = REPLACE('${constants.currentMonth}', '-', '');

diff --git a/definitions/output/core_web_vitals/technologies.js b/definitions/output/core_web_vitals/technologies.js
@@ -9,7 +9,7 @@ publish('technologies', {
     clusterBy: ['geo', 'app', 'rank', 'client'],
     requirePartitionFilter: true
   },
-  tags: ['cwv_tech_report'],
+  tags: ['crux_ready'],
   dependOnDependencyAssertions: true
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}

diff --git a/definitions/output/reports/cwv_tech_adoption.js b/definitions/output/reports/cwv_tech_adoption.js
@@ -8,7 +8,7 @@ publish('cwv_tech_adoption', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_ADOPTION(
   records ARRAY<STRUCT<

diff --git a/definitions/output/reports/cwv_tech_categories.js b/definitions/output/reports/cwv_tech_categories.js
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('cwv_tech_categories', {
   schema: 'reports',
   type: 'table',
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (

diff --git a/definitions/output/reports/cwv_tech_core_web_vitals.js b/definitions/output/reports/cwv_tech_core_web_vitals.js
@@ -8,7 +8,7 @@ publish('cwv_tech_core_web_vitals', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_VITALS(
   records ARRAY<STRUCT<

diff --git a/definitions/output/reports/cwv_tech_lighthouse.js b/definitions/output/reports/cwv_tech_lighthouse.js
@@ -8,7 +8,7 @@ publish('cwv_tech_lighthouse', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE(
   records ARRAY<STRUCT<

diff --git a/definitions/output/reports/cwv_tech_page_weight.js b/definitions/output/reports/cwv_tech_page_weight.js
@@ -8,7 +8,7 @@ publish('cwv_tech_page_weight', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT(
   records ARRAY<STRUCT<

diff --git a/definitions/output/reports/cwv_tech_technologies.js b/definitions/output/reports/cwv_tech_technologies.js
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('cwv_tech_technologies', {
   schema: 'reports',
   type: 'table',
-  tags: ['cwv_tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
 SELECT

diff --git a/definitions/output/reports/reports_dynamic.js b/definitions/output/reports/reports_dynamic.js
@@ -19,7 +19,7 @@ if (iterations.length === 1) {
         protected: true,
         bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
         schema: 'reports',
-        tags: ['crawl_complete', 'crawl_reports']
+        tags: ['crawl_complete']
       }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
@@ -33,7 +33,7 @@ sql.query(ctx, params))
     metrics.forEach(metric => {
       metric.SQL.forEach(sql => {
         operate(metric.id + '_' + sql.type + '_' + params.date, {
-          tags: ['crawl_complete', 'crawl_reports']
+          tags: ['crawl_complete']
         }).queries(ctx => `
 DELETE FROM reports.${metric.id}_${sql.type}
 WHERE date = '${params.date}';

diff --git a/docs/infrastructure.md b/docs/infrastructure.md
@@ -15,8 +15,8 @@ graph LR;
     dataformTrigger_subscription --> dataformTrigger
 
     subgraph Cloud_Scheduler
-        bq_poller_cwv_tech_report[CWV Report Poller Job]
-        bq_poller_cwv_tech_report --> dataformTrigger
+        bq_poller_crux_ready[CrUX Readiness Poller Job]
+        bq_poller_crux_ready --> dataformTrigger
     end
 
     subgraph Dataform

diff --git a/infra/README.md b/infra/README.md
@@ -21,11 +21,23 @@ Request body example with trigger name:
 ```json
 {
   "message": {
-    "name": "cwv_tech_report"
+    "name": "crux_ready"
   }
 }
 ```
 
+Trigger for local development:
+
+```bash
+curl -X POST http://localhost:8080/ \
+  -H "Content-Type: application/json" \
+  -d '{
+    "message": {
+      "name": "crux_ready"
+    }
+  }'
+```
+
 ## Cloud Function for report data exports
 
 [exportReport](https://console.cloud.google.com/functions/details/us-central1/bqExport?env=gen2&authuser=7&project=httparchive) Cloud Run Function
@@ -54,37 +66,7 @@ This function exports reports data to GCS or Firestore.
 }
 ```
 
-## Monitoring
-
-The issues within the pipeline are being tracked using the following alerts:
-
-1. the event trigger processing fails - [Dataform Trigger Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive)
-2. a job in the workflow fails - "[Dataform Workflow Invocation Failed](https://console.cloud.google.com/monitoring/alerting/policies/16526940745374967367?authuser=7&project=httparchive)
-3. the export function fails - [Dataform Export Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive)
-
-Error notifications are sent to [#10x-infra](https://httparchive.slack.com/archives/C030V4WAVL3) Slack channel.
-
-## Local development
-
-To test the function locally run from the function directory:
-
-```bash
-npm run start
-```
-
-Then, in a separate terminal, run the command with the test payload:
-
-```bash
-curl -X POST http://localhost:8080/ \
-  -H "Content-Type: application/json" \
-  -d '{
-    "message": {
-      "name": "cwv_tech_report"
-    }
-  }'
-```
-
-or
+Trigger for local development:
 
 ```bash
 curl -X POST http://localhost:8080/ \
@@ -132,6 +114,26 @@ curl -X POST http://localhost:8080/ \
 }'
 ```
 
+## Monitoring
+
+The issues within the pipeline are being tracked using the following alerts:
+
+1. the event trigger processing fails - [Dataform Trigger Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive)
+2. a job in the workflow fails - "[Dataform Workflow Invocation Failed](https://console.cloud.google.com/monitoring/alerting/policies/16526940745374967367?authuser=7&project=httparchive)
+3. the export function fails - [Dataform Export Function Error](https://console.cloud.google.com/monitoring/alerting/policies/570799173843203905?authuser=7&project=httparchive)
+
+Error notifications are sent to [#10x-infra](https://httparchive.slack.com/archives/C030V4WAVL3) Slack channel.
+
+## Local development
+
+To test the function locally run from the function directory:
+
+```bash
+npm run start
+```
+
+Then, in a separate terminal, run the command with the test trigger payload.
+
 ## Deployment
 
 From project root directory run:

diff --git a/infra/dataform-trigger/index.js b/infra/dataform-trigger/index.js
@@ -3,7 +3,7 @@ const { BigQuery } = require('@google-cloud/bigquery')
 const { getCompilationResults, runWorkflow } = require('./dataform')
 
 const TRIGGERS = {
-  cwv_tech_report: {
+  crux_ready: {
     type: 'poller',
     query: `
 DECLARE previousMonth STRING DEFAULT FORMAT_DATE('%Y%m%d', DATE_SUB(DATE_TRUNC(CURRENT_DATE(), MONTH), INTERVAL 1 MONTH));
@@ -31,7 +31,7 @@ FROM crux, report;
     action: 'runDataformRepo',
     actionArgs: {
       repoName: 'crawl-data',
-      tags: ['cwv_tech_report']
+      tags: ['crux_ready']
     }
   },
   crawl_complete: {
@@ -40,9 +40,7 @@ FROM crux, report;
     actionArgs: {
       repoName: 'crawl-data',
       tags: [
-        'crawl_complete',
-        'crawl_reports',
-        'blink_features_report'
+        'crawl_complete'
       ]
     }
   }
@@ -163,7 +161,7 @@ async function runDataformRepo (args) {
  * Example request payload:
  * {
  *  "message": {
- *     "name": "cwv_tech_report"
+ *     "name": "crux_ready"
  *   }
  * }
  */

diff --git a/infra/tf/function_dataform_trigger.tf b/infra/tf/function_dataform_trigger.tf
@@ -82,10 +82,10 @@ resource "google_pubsub_subscription" "dataform_crawl_complete" {
 
 # Cloud Scheduler Job to trigger CWV Tech Report Dataform workflow
 locals {
-  cwv_tech_report_scheduler_body = <<EOF
+  crux_ready_scheduler_body = <<EOF
 {
   "message": {
-    "name": "cwv_tech_report"
+    "name": "crux_ready"
   }
 }
 EOF
@@ -108,7 +108,7 @@ resource "google_cloud_scheduler_job" "bq-poller-cwv-tech-report" {
   schedule         = "0 */7 8-14 * *"
   time_zone        = "Etc/UTC"
   http_target {
-    body = base64encode(local.cwv_tech_report_scheduler_body)
+    body = base64encode(local.crux_ready_scheduler_body)
     headers = {
       Content-Type = "application/json"
     }