From 826c26b5dce250e0b155edd8ffdc4193da85a0da Mon Sep 17 00:00:00 2001
From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:21:29 +0200
Subject: [PATCH] no drop table

---
 definitions/output/all/reprocess_pages.js    | 7 +++++--
 definitions/output/all/reprocess_requests.js | 7 +++++--
 workflow_settings.yaml                       | 4 ++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/definitions/output/all/reprocess_pages.js b/definitions/output/all/reprocess_pages.js
index dbac9ad..bc60f87 100644
--- a/definitions/output/all/reprocess_pages.js
+++ b/definitions/output/all/reprocess_pages.js
@@ -3,9 +3,9 @@ operate(`all_pages_stable_pre`).tags(
 ).queries(`
 CREATE SCHEMA IF NOT EXISTS all_dev;
 
-DROP TABLE IF EXISTS \`all_dev.pages_stable\`;
+-- DROP TABLE IF EXISTS \`all_dev.pages_stable\`;
 
-CREATE TABLE \`all_dev.pages_stable\`
+CREATE TABLE IF NOT EXISTS \`all_dev.pages_stable\`
 (
   date DATE NOT NULL OPTIONS(description="YYYY-MM-DD format of the HTTP Archive monthly crawl"),
   client STRING NOT NULL OPTIONS(description="Test environment: desktop or mobile"),
@@ -80,6 +80,9 @@ iterations.forEach((iteration, i) => {
   ]).dependencies([
     i === 0 ? "all_pages_stable_pre" : `all_pages_stable_update ${iterations[i - 1].month} ${iterations[i - 1].client}`
   ]).queries(ctx => `
+DELETE FROM \`all_dev.pages_stable\`
+WHERE date = "${iteration.month}";
+
 INSERT INTO \`all_dev.pages_stable\`
 SELECT
   date,
diff --git a/definitions/output/all/reprocess_requests.js b/definitions/output/all/reprocess_requests.js
index 50e06c0..0cd6650 100644
--- a/definitions/output/all/reprocess_requests.js
+++ b/definitions/output/all/reprocess_requests.js
@@ -3,9 +3,9 @@ operate(`all_requests_stable_pre`).tags(
 ).queries(`
 CREATE SCHEMA IF NOT EXISTS all_dev;
 
-DROP TABLE IF EXISTS \`all_dev.requests_stable\`;
+-- DROP TABLE IF EXISTS \`all_dev.requests_stable\`;
 
-CREATE TABLE \`all_dev.requests_stable\`
+CREATE TABLE IF NOT EXISTS \`all_dev.requests_stable\`
 (
   date DATE NOT NULL OPTIONS(description="YYYY-MM-DD format of the HTTP Archive monthly crawl"),
   client STRING NOT NULL OPTIONS(description="Test environment: desktop or mobile"),
@@ -57,6 +57,9 @@ iterations.forEach((iteration, i) => {
   ).dependencies([
     i === 0 ? "all_requests_stable_pre" : `all_requests_stable ${iterations[i - 1].month} ${iterations[i - 1].client}`
   ]).queries(ctx => `
+DELETE FROM \`all_dev.requests_stable\`
+WHERE date = "${iteration.month}";
+
 CREATE TEMP FUNCTION PRUNE_HEADERS(
   jsonObject JSON
 ) RETURNS JSON
diff --git a/workflow_settings.yaml b/workflow_settings.yaml
index 4aeb4eb..867965b 100644
--- a/workflow_settings.yaml
+++ b/workflow_settings.yaml
@@ -5,5 +5,5 @@ defaultAssertionDataset: dataform_assertions
 
 vars:
   placeholder: value  # keeping it to avoid 'empty vars' error
-  env_name: dev  # MUST be commented in main branch, enables processing sampled data
-  today: 2024-09-20  # MUST be commented in main branch, allows processing historical data
+  # env_name: dev  # MUST be commented in main branch, enables processing sampled data
+  # today: 2024-09-20  # MUST be commented in main branch, allows processing historical data