Merge branch 'master' into feat/estoque-conectividade

.github/workflows/dbt-compile.yaml

@@ -1,6 +1,6 @@
 name: DBT Compile
 
-on: [pull_request, push]
+on: [pull_request]
 
 jobs:
   dbt-compile:
@@ -21,11 +21,14 @@ jobs:
     - name: Mount credentials
       env:
         DBT_CREDENTIALS_DEV: ${{ secrets.DBT_CREDENTIALS_DEV }}
-      run: echo "$DBT_CREDENTIALS_DEV" > credentials/dev.json
+      run: echo "$DBT_CREDENTIALS_DEV" > /tmp/credentials.json
 
     - name: Install dbt
       run:  pip install dbt-core dbt-bigquery setuptools
 
+    - name: Install DBT Dependencies
+      run:  dbt deps
+
     - name: Compile dbt models
-      run: dbt compile --profiles-dir . --profile default --target compile
+      run: dbt compile --profiles-dir . --profile default --target dev
 

.github/workflows/sql-linter.yaml

@@ -0,0 +1,96 @@
+name: SQLFluff
+
+on:
+  - pull_request
+
+jobs:
+  lint-models:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Install Python
+        uses: "actions/setup-python@v2"
+        with:
+          python-version: "3.10"
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: '1.7.1'
+
+      - name: Setup a local virtual environment (if no poetry.toml file)
+        run: |
+          poetry config virtualenvs.create true --local
+          poetry config virtualenvs.in-project true --local
+
+      - name: Install the project dependencies
+        run: poetry install --no-interaction
+
+      - name: Add Poetry virtualenv to PATH
+        run: |
+          echo "$(poetry env info --path)/bin" >> $GITHUB_PATH
+
+      - name: Install DBT Dependencies
+        run: "dbt deps"
+
+      - name: Get changed files
+        id: get_file_changes
+        uses: trilom/[email protected]
+        with:
+          output: ' '
+
+      - name: Display changed files
+        run: |
+          echo "Modified files: ${{ steps.get_file_changes.outputs.files_modified }}"
+          echo "Added files: ${{ steps.get_file_changes.outputs.files_added }}"
+
+      - name: Get changed .sql files in /models to lint
+        id: get_files_to_lint
+        shell: bash -l {0}
+        run: |
+          # Set the command in the $() brackets as an output to use in later steps
+          echo "::set-output name=lintees::$(
+          # Issue where grep regular expressions don't work as expected on the
+          # Github Actions shell, check models/ folder
+          echo \
+          $(echo ${{ steps.get_file_changes.outputs.files_modified }} |
+          tr -s ' ' '\n' |
+          grep -E '^models.*[.]sql$' |
+          tr -s '\n' ' ') \
+          $(echo ${{ steps.get_file_changes.outputs.files_added }} |
+          tr -s ' ' '\n' |
+          grep -E '^models.*[.]sql$' |
+          tr -s '\n' ' ')
+          )"
+      
+      - name: Lint each SQL file individually
+        run: |
+          # Loop through each changed SQL file and lint them individually
+          for file in ${{ steps.get_file_changes.outputs.files_modified }} ${{ steps.get_file_changes.outputs.files_added }}; do
+            if [[ $file == *.sql ]]; then
+              echo "Linting $file"
+              sqlfluff lint failure $file || exit_code=$?
+            fi
+          done
+        shell: bash
+
+      - name: Fail if any file had a linting error
+        run: |
+          if [[ -n "$exit_code" ]]; then
+            exit $exit_code
+          fi
+
+      - name: Lint dbt models
+        id: sqlfluff_json
+        if: steps.get_files_to_lint.outputs.lintees != ''
+        shell: bash -l {0}
+        run: sqlfluff lint ./models/ > annotations.json
+
+      - name: Annotate
+        uses: yuzutech/[email protected]
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          title: "SQLFluff Lint"
+          input: "./annotations.json"

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files # prevents adding large files
+      - id: detect-private-key # detects private keys
+      - id: fix-byte-order-marker # fixes BOM

.sqlfluff

@@ -0,0 +1,54 @@
+[sqlfluff]
+
+dialect = bigquery
+
+templater = dbt
+
+runaway_limit = 10
+
+max_line_length = 120
+
+indent_unit = space
+large_file_skip_byte_limit = 30000
+
+[sqlfluff:indentation]
+
+tab_space_size = 4
+
+[sqlfluff:layout:type:comma]
+
+spacing_before = touch
+
+line_position = trailing
+
+[sqlfluff:rules:capitalisation.keywords] 
+
+capitalisation_policy = upper
+
+[sqlfluff:rules:aliasing.table]
+
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.column]
+
+aliasing = explicit
+
+[sqlfluff:rules:aliasing.expression]
+
+allow_scalar = False
+
+[sqlfluff:rules:capitalisation.identifiers]
+
+extended_capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.functions]
+
+capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.literals]
+
+capitalisation_policy = lower
+
+[sqlfluff:rules:ambiguous.column_references]  # Number in group by
+
+group_by_and_order_by_style = implicit