incremental todos : full refresh for recurring tasks

luutuankiet · Dec 6, 2024 · 1088b29 · 1088b29
1 parent b006087
commit 1088b29
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 7 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,8 +14,8 @@
     "python.venvFolders": [
         "${workspaceFolder}/.venv"
     ],
-    "python.envFile": "${workspaceFolder}/.env",
-    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+    "python.envFile": "${workspaceFolder}/../.env",
+    "python.defaultInterpreterPath": "${workspaceFolder}/../.venv/bin/python",
 
     "python.terminal.activateEnvironment": true,
     "terminal.integrated.inheritEnv": true,
@@ -27,7 +27,7 @@
         "/**/.github/**/*.yml": "github-actions-workflow"
     },
     "terminal.integrated.environmentChangesRelaunch": true,
-    "dbt.dbtPythonPathOverride": "${workspaceFolder}/.venv/bin/python",
+    "dbt.dbtPythonPathOverride": "${workspaceFolder}/../.venv/bin/python",
 
 
 

diff --git a/models/marts/core/fact_todos.sql b/models/marts/core/fact_todos.sql
@@ -23,7 +23,8 @@ FROM
     source
 
 {% if is_incremental() %}
-  WHERE  todo_modifiedtime >= (select coalesce(max(todo_modifiedtime),'1900-01-01 00:00:00') from {{ this }} )
-OR
+  WHERE 
+  _modified_time >= (select coalesce(max(_modified_time),'1900-01-01 00:00:00') from {{ this }} )
+  OR
   todo_modifiedtime IS NULL
 {% endif %}
diff --git a/models/staging/stg_todos.sql b/models/staging/stg_todos.sql
@@ -256,13 +256,19 @@ joined AS (
 )
 SELECT
     {{ dbt_utils.generate_surrogate_key(['todo_id']) }} AS todo_key,
-    *
+    *,
+    {{dbt_date.now("Asia/Ho_Chi_Minh")}} as _modified_time
 FROM
     joined
 
 {% if is_incremental() %}
   WHERE 
-  todo_completedtime >= todo_modifiedtime -- address this bug tasks completed dont get updated in modified time
+  (-- address this bug tasks completed dont get updated in modified time
+    todo_completedtime >= todo_modifiedtime 
+    AND todo_completedtime::date >= {{dbt_date.n_days_ago(7)}}
+    )
   OR
+   todo_derived__is_repeat is True -- full refresh all habit tasks 
+  OR 
   todo_modifiedtime >= (select coalesce(max(todo_modifiedtime),'1900-01-01 00:00:00') from {{ this }} )
 {% endif %}