test(dbt-fal): add source tests (#698)

* Add source freshness project

* Add behave feature for testing sources

* Add TODO tags to some integration tests

.github/workflows/test_integration_adapter.yml

@@ -71,7 +71,7 @@ jobs:
           python -m venv .venv
           source .venv/bin/activate
           pip install --upgrade pip
-          pip install build wheel
+          pip install build wheel 'fal[${{ matrix.profile }}]'
 
           if [[ -z '${{ matrix.adapter_version }}' ]]
           then
@@ -171,10 +171,11 @@ jobs:
           UUID=$(uuidgen | head -c8)
           export DB_NAMESPACE="${{ github.run_id }}_${UUID}"
 
-          BEHAVE_TAGS=""
+          BEHAVE_TAGS="--tags=-TODO-${{ matrix.profile }}"
+
           if [[ '${{ matrix.teleport }}' != 'true' ]]
           then
-            BEHAVE_TAGS="--tags=-teleport"
+            BEHAVE_TAGS="$BEHAVE_TAGS --tags=-teleport"
           fi
 
           if [[ '${{ matrix.cloud }}' != 'true' ]]

adapter/integration_tests/features/source.feature

@@ -0,0 +1,30 @@
+@TODO-duckdb
+@TODO-bigquery
+Feature: dbt-fal can query sources
+  Background: Project Setup
+    Given the project source_freshness
+
+  Scenario: Run a Python model that queries a source in local environment
+    When the following shell command is invoked:
+      """
+      python $baseDir/load_freshness_table.py $baseDir $profilesDir
+      """
+    And the following shell command is invoked:
+      """
+      dbt run --profiles-dir $profilesDir --project-dir $baseDir
+      """
+    Then the following models are calculated in order:
+      | model_a |
+
+  @cloud
+  Scenario: Run a Python model that queries a source with Isolate Cloud
+    When the following shell command is invoked:
+      """
+      python $baseDir/load_freshness_table.py $baseDir $profilesDir
+      """
+    And the following shell command is invoked:
+      """
+      dbt run --profiles-dir $profilesDir --project-dir $baseDir -t prod
+      """
+    Then the following models are calculated in order:
+      | model_a |

adapter/integration_tests/projects/source_freshness/dbt_project.yml

@@ -0,0 +1,17 @@
+name: "source_test"
+version: "1.0.0"
+config-version: 2
+profile: "fal_test"
+
+model-paths: ["models"]
+seed-paths: ["data"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "{{ env_var('temp_dir') }}/target"
+
+vars:
+  fal-scripts-path: "fal_scripts"
+
+models:
+  +schema: custom

adapter/integration_tests/projects/source_freshness/fal_project.yml

@@ -0,0 +1,8 @@
+environments:
+  - name: not-funny
+    type: venv
+
+  - name: funny
+    type: venv
+    requirements:
+      - pyjokes==0.6.0

adapter/integration_tests/projects/source_freshness/load_freshness_table.py

@@ -0,0 +1,22 @@
+from datetime import datetime as dt, timezone as tz
+from pandas import DataFrame
+from fal import FalDbt
+from sys import argv
+
+project_dir = argv[1] if len(argv) >= 2 else "."
+profiles_dir = argv[2] if len(argv) >= 3 else ".."
+
+faldbt = FalDbt(project_dir=project_dir, profiles_dir=profiles_dir)
+
+# 10 rows
+df = DataFrame({"loaded_at": dt.now(tz=tz.utc).isoformat(), "info": range(0, 10)})
+
+print(df)
+faldbt.write_to_source(df, "freshness_test", "freshness_table", mode="overwrite")
+
+from time import sleep
+
+# Let BigQuery cache load it
+sleep(5)
+
+print("Loaded")

adapter/integration_tests/projects/source_freshness/models/model_a.py

@@ -0,0 +1,8 @@
+def model(dbt, fal):
+    dbt.config(materialized="table")
+    dbt.config(fal_environment="funny")
+    import pyjokes
+    joke = pyjokes.get_joke()
+    df = dbt.source("freshness_test", "freshness_table")
+    df["my_joke"] = joke
+    return df

adapter/integration_tests/projects/source_freshness/models/schema.yml

@@ -0,0 +1,19 @@
+version: 2
+
+sources:
+  - name: freshness_test
+    database: "{{ env_var('DBT_DATABASE', 'test') }}"
+    schema: "{{ env_var('DBT_SCHEMA', 'dbt_fal_custom') }}"
+    freshness:
+      warn_after: { "count": 5, "period": minute }
+      error_after: { "count": 30, "period": minute }
+    tables:
+      - name: "freshness_table"
+        loaded_at_field: "current_timestamp"
+        columns:
+          - name: info
+            tests:
+              - unique
+
+models:
+  - name: model_a