From b0aed550bf2b801ded4810767d5e8967cc3fac7e Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Tue, 3 Dec 2024 00:35:26 +0000 Subject: [PATCH] added GHA workflows --- .github/actions/setup-env/action.yml | 58 ++++++++++++++ .github/pull_request_template.md | 106 ++++++++++++++++++++++++ .github/workflows/deployment.sh | 19 +++++ .github/workflows/gh_deploy.yml | 115 +++++++++++++++++++++++++++ .github/workflows/slim_CI.yml | 93 ++++++++++++++++++++++ 5 files changed, 391 insertions(+) create mode 100644 .github/actions/setup-env/action.yml create mode 100644 .github/pull_request_template.md create mode 100755 .github/workflows/deployment.sh create mode 100644 .github/workflows/gh_deploy.yml create mode 100644 .github/workflows/slim_CI.yml diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 0000000..344aaf6 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,58 @@ +# .github/actions/setup-env/action.yml +name: "Setup Environment" +description: "Setup environment variables from .env.bootstrap and secrets" + +inputs: + PROD_ENV: + description: "The secret containing environment variables" + required: true + +runs: + using: "composite" + steps: + - name: Bootstrap environment variables + shell: bash + run: | + # Initialize the environment variables using bootstrap_env.sh + . ./bootstrap_env.sh + + # Check if .env.bootstrap file exists + if [ -f .env.bootstrap ]; then + # Process each line in the .env.bootstrap file + while IFS= read -r line; do + # Skip empty lines and lines starting with # + if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + # Export the variable to $GITHUB_ENV, removing surrounding quotes if present + clean_line=$(echo "$line" | awk -F= '{print $1 "=" substr($2, 2, length($2)-2)}') + echo "$clean_line" >> $GITHUB_ENV + fi + done < .env.bootstrap + else + echo ".env.bootstrap file not found" + fi + + - name: Create .env file from secret + shell: bash + run: | + # Decode and create .env file + echo "${{ inputs.PROD_ENV }}" > .env + + - name: Dynamically Mask Environment Variables + shell: bash + run: | + # Read the .env file line by line + while IFS= read -r line; do + # Skip empty lines and lines starting with # + if [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + # Extract variable name and value + var_name=$(echo "$line" | awk -F= '{print $1}') + var_value=$(echo "$line" | awk -F= '{print $2}' | sed 's/^"\(.*\)"$/\1/') + + # Mask the variable value + echo "::add-mask::$var_value" + + # Export the variable to $GITHUB_ENV without quotes + clean_line="$var_name=$var_value" + echo "$clean_line" >> "$GITHUB_ENV" + fi + done < .env \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..848a653 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,106 @@ + +<!--- + +Provide a short summary in the Title above. Examples of good PR titles: + +* "Feature: add so-and-so models" + +* "Fix: deduplicate such-and-such" + +* "Update: dbt version 0.13.0" + +--> + +## Description & motivation + +<!--- + +Describe your changes, and why you're making them. Is this linked to an open + +issue, a Trello card, or another pull request? Link it here. + +--> + +## To-do before merge + +<!--- + +(Optional -- remove this section if not needed) + +Include any notes about things that need to happen before this PR is merged, e.g.: + +- [ ] Change the base branch + +- [ ] Update dbt Cloud jobs + +- [ ] Ensure PR #56 is merged + +--> + +## Screenshots: + +<!--- + +Include a screenshot of the relevant section of the updated DAG. You can access + +your version of the DAG by running `dbt docs generate && dbt docs serve`. + +--> + +## Validation of models: + +<!--- + +Include any output that confirms that the models do what is expected. This might + +be a link to an in-development dashboard in your BI tool, or a query that + +compares an existing model with a new one. + +--> + +## Changes to existing models: + +<!--- + +Include this section if you are changing any existing models. Link any related + +pull requests on your BI tool, or instructions for merge (e.g. whether old + +models should be dropped after merge, or whether a full-refresh run is required) + +--> + +## Checklist: + +<!--- + +This checklist is mostly useful as a reminder of small things that can easily be + +forgotten – it is meant as a helpful tool rather than hoops to jump through. + +Put an `x` in all the items that apply, make notes next to any that haven't been + +addressed, and remove any items that are not relevant to this PR. + +--> + +- [ ] My pull request represents one logical piece of work. + +- [ ] My commits are related to the pull request and look clean. + +- [ ] My SQL follows the style guide. + +- [ ] I have materialized my models appropriately. + +- [ ] I have added appropriate tests and documentation to any new models. + +- [ ] I have updated the README file. + +{%- if project.warehouse == 'redshift' %} + +- [ ] I have added sort and dist keys to models materialized as tables. + +- [ ] I have validated the SQL in any late-binding views. + +{% endif %} diff --git a/.github/workflows/deployment.sh b/.github/workflows/deployment.sh new file mode 100755 index 0000000..706dcfd --- /dev/null +++ b/.github/workflows/deployment.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +WORKDIR=$(pwd) + +. $WORKDIR/.venv/bin/activate +. $WORKDIR/bootstrap_env.sh + +# Array of tmux session names +sessions=("dagster" "loader" "gtd_search") + +for session in "${sessions[@]}"; do + # Check if the session exists + if tmux has-session -t "$session" 2>/dev/null; then + # Session exists, kill the old one + tmux kill-session -t "$session" + fi + # Create a new session + tmux new-session -s "$session" -d +done \ No newline at end of file diff --git a/.github/workflows/gh_deploy.yml b/.github/workflows/gh_deploy.yml new file mode 100644 index 0000000..a3c3f00 --- /dev/null +++ b/.github/workflows/gh_deploy.yml @@ -0,0 +1,115 @@ +name: deploy +on: + workflow_dispatch: + pull_request: + types: + - closed + branches: + - main + - master + +env: + TARGET_SCHEMA: prod + +jobs: + deploy_git: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + env: + SERVER_HOST_KEY: ${{ secrets.SERVER_HOST_KEY }} + CLIENT_SECRET_KEY: ${{ secrets.CLIENT_SECRET_KEY }} + HOSTNAME: ${{ secrets.HOSTNAME }} + SSH_PORT: ${{ secrets.SSH_PORT }} + steps: + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "$SERVER_HOST_KEY" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + echo "$CLIENT_SECRET_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + + - name: Update Server + run: | + eval "$(ssh-agent -s)" + ssh-add ~/.ssh/id_ed25519 + ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes root@$HOSTNAME -p $SSH_PORT << 'EOF' + cd /workspaces/gtd_ETL + git pull origin main + git submodule update --init --remote --merge + . ./bootstrap_env.sh + . ./venv/bin/activate + pip install -r requirements.txt + dbt deps + make deploy + EOF + + deploy_lightdash: + runs-on: ubuntu-latest + needs: deploy_git + + + + steps: + - uses: actions/checkout@v3 + + - uses: ./.github/actions/setup-env + with: + PROD_ENV: ${{ secrets.PROD_ENV }} + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + ~/.local + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - uses: actions/setup-python@v4 + with: + python-version: "3.11.x" + + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Cache npm packages + uses: actions/cache@v3 + with: + path: | + ~/.npm + ~/.cache/npm + node_modules + key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm- + + - uses: actions/setup-node@v3.4.1 + with: + node-version: '20' + + - name: Get lightdash version + uses: sergeysova/jq-action@v2 + id: version + env: + LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }} + with: + cmd: curl -s "${LIGHTDASH_URL}/api/v1/health" | jq -r '.results.version' + + - name: Install npm dependencies + run: npm install -g "@lightdash/cli@${{ steps.version.outputs.value }}" || npm install -g @lightdash/cli@latest + + + + - name: Lightdash CLI deploy + env: + LIGHTDASH_API_KEY: ${{ secrets.LIGHTDASH_API_KEY }} + LIGHTDASH_PROJECT: ${{ secrets.LIGHTDASH_PROJECT }} + LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }} + GOOGLE_APPLICATION_CREDENTIALS: '/tmp/googlecredentials.json' + run: lightdash deploy diff --git a/.github/workflows/slim_CI.yml b/.github/workflows/slim_CI.yml new file mode 100644 index 0000000..1b7bb69 --- /dev/null +++ b/.github/workflows/slim_CI.yml @@ -0,0 +1,93 @@ +name: slim_CI + +# for slim CIs all dbt and lightdash runs are built on dev schema; just need to overwrite TARGET_SCHEMA env which lives in the PROD_ENV secret. +on: + push: + branches-ignore: [ "main", "master"] + + +jobs: + validate_dbt_models: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: 8BitJonny/gh-get-current-pr@3.0.0 + id: PR + + - name: Set var PR_schema + run: echo "PR_schema=${{ env.schema_var }}" >> $GITHUB_ENV + if: ${{ steps.PR.outputs.pr_found == 'true' }} + env: + schema_var: '{\"schema_name\": \"ci_PR_${{ steps.PR.outputs.number }}\"}' + + - name: debug + run: echo "${{ env.PR_schema }}" + + + - uses: actions/checkout@v3 + + - uses: ./.github/actions/setup-env + with: + PROD_ENV: ${{ secrets.PROD_ENV }} + + + - name: Cache Python packages + uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + ~/.local + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - uses: actions/setup-python@v4 + with: + python-version: "3.11.x" + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Cache npm packages + uses: actions/cache@v3 + with: + path: | + ~/.npm + ~/.cache/npm + node_modules + key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm- + + - uses: actions/setup-node@v3 + with: + node-version: "20" + + - name: Get lightdash version + uses: sergeysova/jq-action@v2 + id: version + env: + LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }} + with: + cmd: curl -s "${LIGHTDASH_URL}/api/v1/health" | jq -r '.results.version' + + + - name: Install npm dependencies + run: npm install -g "@lightdash/cli@${{ steps.version.outputs.value }}" || npm install -g @lightdash/cli@latest + + + - name: dbt build models + run: dbt build --target ci --vars '${{env.PR_schema}}' + + + - name: validate LD + id: validate-lightdash + run: lightdash validate --target ci --vars '${{env.PR_schema}}' + env: + LIGHTDASH_API_KEY: ${{ secrets.LIGHTDASH_API_KEY }} + LIGHTDASH_PROJECT: ${{ secrets.LIGHTDASH_PROJECT }} + LIGHTDASH_URL: ${{ secrets.LIGHTDASH_URL }} \ No newline at end of file