Merge pull request #9 from fivetran/jm-updates

Jm updates
fivetran · Mar 29, 2021 · 2021bfc · 2021bfc
2 parents c6b070f + 6243caf
commit 2021bfc
Show file tree

Hide file tree

Showing 45 changed files with 506 additions and 95 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,60 @@
+version: 2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/python:3.7.9-stretch
+
+    steps:
+      - checkout
+
+      - run:
+          run: setup_creds
+          command: |
+            echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json
+      - run:
+          name: "Setup dbt"
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install --upgrade pip setuptools
+            pip install -r integration_tests/requirements.txt
+            mkdir -p ~/.dbt
+            cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+      - run:
+          name: "Run Tests - Redshift"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target redshift --full-refresh
+            dbt run --target redshift --full-refresh
+            dbt test --target redshift
+      - run:
+          name: "Run Tests - Snowflake"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target snowflake --full-refresh
+            dbt run --target snowflake --full-refresh
+            dbt test --target snowflake
+      - run:
+          name: "Run Tests - BigQuery"
+          environment:
+              GCLOUD_SERVICE_KEY_PATH: "/home/circleci/gcloud-service-key.json"
+
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target bigquery --full-refresh
+            dbt run --target bigquery --full-refresh
+            dbt test --target bigquery
+      - save_cache:
+          key: deps2-{{ .Branch }}
+          paths:
+            - "venv"
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Asana ([docs](https://fivetran-dbt-asana.netlify.app/#!/overview))
 
-This package models Asana data from [Fivetran's connector](https://fivetran.com/docs/applications/asana). It uses data in the format described by [this ERD](https://docs.google.com/presentation/d/14m2L2aYGmt0IXseExR80FlEO-7fxjBKfoALR2jVh0G8/edit).
+This package models Asana data from [Fivetran's connector](https://fivetran.com/docs/applications/asana). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/asana#schemainformation).
 
 This package enables you to better understand tasks and how they're being worked on in Asana. Its primary focus is to enhance the task table and other core objects that relate to tasks: users, projects, teams, and tags. Each of these objects is enriched with metrics that reflect the volume and breadth of work being done now and the velocity of work that has been completed. Moreover, the daily metrics table lays out a timeline of task creations and completions for understanding the overall pace of deliverables at the organization.
 
@@ -12,12 +12,12 @@ This package contains transformation models, designed to work simultaneously wit
 
 | **model**                | **description**                                                                                                                                |
 | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- |
-| asana_task             | Each record represents an Asana task, enriched with data about its assignee, projects, sections, tasks, teams, tags, parent task, comments, followers, and activity. |      
-| asana_user             | Each record represents an Asana user, enriched with metrics about their completed tasks, open tasks, and the projects they work on. Also includes data about the user's most recently completed task and their next due task. |
-| asana_project          | Each record represents an Asana project, enriched with metrics about their completed tasks, open tasks, and the users involved in the project. Also includes data about the project's most recently completed task and next due tasks. |
-| asana_team             | Each record represents an Asana team, enriched with data about their completed tasks, open tasks, their projects, and the users involved with the team. |
-| asana_tag              | Each record represents an Asana tag, enriched with metrics about open and completed tasks associated with the tag. |
-| asana_daily_metrics    | Each record represents a single day, enriched with metrics about tasks opened at created that day. |
+| [asana__task](https://github.com/fivetran/dbt_asana/blob/master/models/asana__task.sql)             | Each record represents an Asana task, enriched with data about its assignee, projects, sections, tasks, teams, tags, parent task, comments, followers, and activity. |      
+| [asana__user](https://github.com/fivetran/dbt_asana/blob/master/models/asana__user.sql)             | Each record represents an Asana user, enriched with metrics about their completed tasks, open tasks, and the projects they work on. Also includes data about the user's most recently completed task and their next due task. |
+| [asana_project](https://github.com/fivetran/dbt_asana/blob/master/models/asana__project.sql)          | Each record represents an Asana project, enriched with metrics about their completed tasks, open tasks, and the users involved in the project. Also includes data about the project's most recently completed task and next due tasks. |
+| [asana__team](https://github.com/fivetran/dbt_asana/blob/master/models/asana__team.sql)             | Each record represents an Asana team, enriched with data about their completed tasks, open tasks, their projects, and the users involved with the team. |
+| [asana__tag](https://github.com/fivetran/dbt_asana/blob/master/models/asana__tag.sql)              | Each record represents an Asana tag, enriched with metrics about open and completed tasks associated with the tag. |
+| [asana__daily_metrics](https://github.com/fivetran/dbt_asana/blob/master/models/asana__daily_metrics.sql)    | Each record represents a single day, enriched with metrics about tasks opened at created that day. |
 
 
 ## Installation Instructions
@@ -37,6 +37,22 @@ vars:
     asana_database: your_database_name
     asana_schema: your_schema_name 
 ```
+### Changing the Build Schema
+By default this package will build the Asana staging models within a schema titled (<target_schema> + `_stg_asana`) and the Asana final models with a schema titled (<target_schema> + `_asana`) in your target database. If this is not where you would like your modeled Asana data to be written to, add the following configuration to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+...
+models:
+  asana:
+    +schema: my_new_schema_name # leave blank for just the target_schema
+  asana_source:
+    +schema: my_new_schema_name # leave blank for just the target_schema
+```
+
+## Database support
+This package is compatible with BigQuery, Snowflake, and Redshift.
 
 ## Contributions
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,26 +1,27 @@
 config-version: 2
 
 name: 'asana'
-version: '0.2.0'
+version: '0.3.0'
 
-require-dbt-version: [">=0.18.0", "<0.19.0"]
+require-dbt-version: [">=0.18.0", "<0.20.0"]
 
 models:
     asana:
+        +schema: asana
         materialized: table
         intermediate:
             materialized: ephemeral 
 
 vars:
     asana: 
-        project_task: "{{ ref('stg_asana_project_task') }}"
-        project: "{{ ref('stg_asana_project') }}"
-        section: "{{ ref('stg_asana_section') }}"
-        story: "{{ ref('stg_asana_story') }}"
-        tag: "{{ ref('stg_asana_tag') }}"
-        task_follower: "{{ ref('stg_asana_task_follower') }}"
-        task_section: "{{ ref('stg_asana_task_section') }}"
-        task_tag: "{{ ref('stg_asana_task_tag') }}"
-        task: "{{ ref('stg_asana_task') }}"
-        team: "{{ ref('stg_asana_team') }}"
-        user: "{{ ref('stg_asana_user') }}"
+        project_task:   "{{ ref('stg_asana__project_task') }}"
+        project:        "{{ ref('stg_asana__project') }}"
+        section:        "{{ ref('stg_asana__section') }}"
+        story:          "{{ ref('stg_asana__story') }}"
+        tag:            "{{ ref('stg_asana__tag') }}"
+        task_follower:  "{{ ref('stg_asana__task_follower') }}"
+        task_section:   "{{ ref('stg_asana__task_section') }}"
+        task_tag:       "{{ ref('stg_asana__task_tag') }}"
+        task:           "{{ ref('stg_asana__task') }}"
+        team:           "{{ ref('stg_asana__team') }}"
+        user:           "{{ ref('stg_asana__user') }}"
diff --git a/docs/catalog.json b/docs/catalog.json