no drop table

HTTPArchive · Oct 8, 2024 · 826c26b · 826c26b
1 parent 65f098c
commit 826c26b
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 6 deletions.
diff --git a/definitions/output/all/reprocess_pages.js b/definitions/output/all/reprocess_pages.js
@@ -3,9 +3,9 @@ operate(`all_pages_stable_pre`).tags(
 ).queries(`
 CREATE SCHEMA IF NOT EXISTS all_dev;
 
-DROP TABLE IF EXISTS \`all_dev.pages_stable\`;
+-- DROP TABLE IF EXISTS \`all_dev.pages_stable\`;
 
-CREATE TABLE \`all_dev.pages_stable\`
+CREATE TABLE IF NOT EXISTS \`all_dev.pages_stable\`
 (
   date DATE NOT NULL OPTIONS(description="YYYY-MM-DD format of the HTTP Archive monthly crawl"),
   client STRING NOT NULL OPTIONS(description="Test environment: desktop or mobile"),
@@ -80,6 +80,9 @@ iterations.forEach((iteration, i) => {
   ]).dependencies([
     i === 0 ? "all_pages_stable_pre" : `all_pages_stable_update ${iterations[i - 1].month} ${iterations[i - 1].client}`
   ]).queries(ctx => `
+DELETE FROM \`all_dev.pages_stable\`
+WHERE date = "${iteration.month}";
+
 INSERT INTO \`all_dev.pages_stable\`
 SELECT
   date,

diff --git a/definitions/output/all/reprocess_requests.js b/definitions/output/all/reprocess_requests.js
@@ -3,9 +3,9 @@ operate(`all_requests_stable_pre`).tags(
 ).queries(`
 CREATE SCHEMA IF NOT EXISTS all_dev;
 
-DROP TABLE IF EXISTS \`all_dev.requests_stable\`;
+-- DROP TABLE IF EXISTS \`all_dev.requests_stable\`;
 
-CREATE TABLE \`all_dev.requests_stable\`
+CREATE TABLE IF NOT EXISTS \`all_dev.requests_stable\`
 (
   date DATE NOT NULL OPTIONS(description="YYYY-MM-DD format of the HTTP Archive monthly crawl"),
   client STRING NOT NULL OPTIONS(description="Test environment: desktop or mobile"),
@@ -57,6 +57,9 @@ iterations.forEach((iteration, i) => {
   ).dependencies([
     i === 0 ? "all_requests_stable_pre" : `all_requests_stable ${iterations[i - 1].month} ${iterations[i - 1].client}`
   ]).queries(ctx => `
+DELETE FROM \`all_dev.requests_stable\`
+WHERE date = "${iteration.month}";
+
 CREATE TEMP FUNCTION PRUNE_HEADERS(
   jsonObject JSON
 ) RETURNS JSON

diff --git a/workflow_settings.yaml b/workflow_settings.yaml
@@ -5,5 +5,5 @@ defaultAssertionDataset: dataform_assertions
 
 vars:
   placeholder: value  # keeping it to avoid 'empty vars' error
-  env_name: dev  # MUST be commented in main branch, enables processing sampled data
-  today: 2024-09-20  # MUST be commented in main branch, allows processing historical data
+  # env_name: dev  # MUST be commented in main branch, enables processing sampled data
+  # today: 2024-09-20  # MUST be commented in main branch, allows processing historical data