feat: allow to host dbt docs on cluster (#892)

Adds the (optional, off by default) ability to run dbt docs in the Tutor context so that custom dbt packages and Superset dashboards / charts will be represented in the documentation and be correct for what is actually being run in the environment.
openedx · Aug 20, 2024 · 87f8100 · 87f8100
1 parent b9727df
commit 87f8100
Show file tree

Hide file tree

Showing 17 changed files with 368 additions and 89 deletions.
diff --git a/.ci/config.yml b/.ci/config.yml
@@ -101,3 +101,4 @@ ASPECTS_ENABLE_EVENT_BUS_PRODUCER: true
 SUPERSET_DASHBOARD_LOCALES:
 - en
 - fr_CA
+RUN_ASPECTS_DOCS: true
diff --git a/tutoraspects/patches/caddyfile b/tutoraspects/patches/caddyfile
@@ -10,3 +10,9 @@
     import proxy "superset:{{SUPERSET_PORT}}"
 }
 {% endif %}
+
+{% if RUN_ASPECTS_DOCS %}
+{{ DBT_HOST }}{$default_site_port} {
+    import proxy "aspects-documentation:8090"
+}
+{% endif%}
diff --git a/tutoraspects/patches/k8s-deployments b/tutoraspects/patches/k8s-deployments
@@ -589,3 +589,86 @@ spec:
           configMap:
             name: openedx-config
 {% endif %}
+
+{% if RUN_ASPECTS_DOCS %}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aspects-docs
+  labels:
+    app.kubernetes.io/name: aspects-docs
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: aspects-docs
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: aspects-docs
+    spec:
+      containers:
+      - name: aspects-docs
+        args:
+          - bash
+          - /app/aspects/scripts/docs.sh
+        env:
+          - name: VENV_DIR
+            value: /opt/venv
+          - name: ASPECTS_EVENT_SINK_DATABASE
+            value: {{ASPECTS_EVENT_SINK_DATABASE}}
+          - name: ASPECTS_XAPI_DATABASE
+            value: {{ASPECTS_XAPI_DATABASE}}
+          - name: CLICKHOUSE_CLUSTER_NAME
+            value: {{CLICKHOUSE_CLUSTER_NAME}}
+          - name: DBT_STATE
+            value: {{ DBT_STATE_DIR }}
+          - name: ASPECTS_DATA_TTL_EXPRESSION
+            value: "{{ ASPECTS_DATA_TTL_EXPRESSION }}"
+          - name: DBT_PROFILE_TARGET_DATABASE
+            value: "{{ DBT_PROFILE_TARGET_DATABASE }}"
+          - name: DBT_PROFILES_DIR
+            value: {{ DBT_PROFILES_DIR }}
+          - name: DBT_BRANCH
+            value: "{{ DBT_BRANCH }}"
+          - name: DBT_REPOSITORY
+            value: "{{ DBT_REPOSITORY }}"
+          - name: DBT_SSH_KEY
+            value: "{{ DBT_SSH_KEY }}"
+        image: {{ DOCKER_IMAGE_ASPECTS }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          runAsUser: 0
+        volumeMounts:
+          - mountPath: /app
+            name: app
+          - mountPath: /app/aspects/dbt
+            name: dbt
+          - mountPath: /app/aspects/scripts
+            name: scripts
+          - mountPath: /app/aspects/migrations/
+            name: migrations
+          - mountPath: /app/aspects/migrations/alembic
+            name: alembic
+          - mountPath: /app/aspects/migrations/alembic/versions
+            name: versions
+      volumes:
+        - name: scripts
+          configMap:
+            name: aspects-scripts
+            defaultMode: 0777
+        - name: dbt
+          configMap:
+            name: aspects-dbt
+        - name: app
+          emptyDir: {}
+        - name: migrations
+          configMap:
+            name: aspects-migrations
+        - name: alembic
+          configMap:
+            name: aspects-migrations-alembic
+        - name: versions
+          configMap:
+            name: aspects-migrations-alembic-versions
+{% endif %}
diff --git a/tutoraspects/patches/k8s-jobs b/tutoraspects/patches/k8s-jobs
@@ -14,10 +14,26 @@ spec:
         env:
           - name: VENV_DIR
             value: /opt/venv
-          - name: XAPI_SCHEMA
-            value: {{ ASPECTS_XAPI_DATABASE }}
+          - name: ASPECTS_EVENT_SINK_DATABASE
+            value: {{ASPECTS_EVENT_SINK_DATABASE}}
+          - name: ASPECTS_XAPI_DATABASE
+            value: {{ASPECTS_XAPI_DATABASE}}
+          - name: CLICKHOUSE_CLUSTER_NAME
+            value: {{CLICKHOUSE_CLUSTER_NAME}}
           - name: DBT_STATE
             value: {{ DBT_STATE_DIR }}
+          - name: ASPECTS_DATA_TTL_EXPRESSION
+            value: "{{ ASPECTS_DATA_TTL_EXPRESSION }}"
+          - name: DBT_PROFILE_TARGET_DATABASE
+            value: "{{ DBT_PROFILE_TARGET_DATABASE }}"
+          - name: DBT_PROFILES_DIR
+            value: {{ DBT_PROFILES_DIR }}
+          - name: DBT_BRANCH
+            value: "{{ DBT_BRANCH }}"
+          - name: DBT_REPOSITORY
+            value: "{{ DBT_REPOSITORY }}"
+          - name: DBT_SSH_KEY
+            value: "{{ DBT_SSH_KEY }}"
         image: {{ DOCKER_IMAGE_ASPECTS }}
         securityContext:
           allowPrivilegeEscalation: false
@@ -35,12 +51,7 @@ spec:
             name: alembic
           - mountPath: /app/aspects/migrations/alembic/versions
             name: versions
-          - mountPath: {{ DBT_STATE_DIR }}
-            name: dbt-state
       volumes:
-        - name: dbt-state
-          persistentVolumeClaim:
-            claimName: aspects-dbt
         - name: scripts
           configMap:
             name: aspects-scripts

diff --git a/tutoraspects/patches/k8s-services b/tutoraspects/patches/k8s-services
@@ -52,4 +52,21 @@ spec:
       protocol: TCP
   selector:
     app.kubernetes.io/name: superset
-{% endif %}
+{% endif %}
+
+{% if RUN_ASPECTS_DOCS %}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: aspects
+  labels:
+    app.kubernetes.io/name: aspects-docs
+spec:
+  type: ClusterIP
+  ports:
+    - port: 7000
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: aspects-docs
+{% endif %}
diff --git a/tutoraspects/patches/k8s-volumes b/tutoraspects/patches/k8s-volumes
@@ -1,18 +1,3 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: aspects-dbt
-  labels:
-    app.kubernetes.io/component: volume
-    app.kubernetes.io/name: aspects-dbt
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10M
-
 {% if RUN_CLICKHOUSE %}
 ---
 apiVersion: v1

diff --git a/tutoraspects/patches/local-docker-compose-jobs-services b/tutoraspects/patches/local-docker-compose-jobs-services
@@ -2,15 +2,23 @@ aspects-job:
   user: root
   image: {{ DOCKER_IMAGE_ASPECTS }}
   environment:
-    - XAPI_SCHEMA={{ ASPECTS_XAPI_DATABASE }}
+    VENV_DIR: /opt/venv
+    ASPECTS_EVENT_SINK_DATABASE: {{ASPECTS_EVENT_SINK_DATABASE}}
+    ASPECTS_XAPI_DATABASE: {{ASPECTS_XAPI_DATABASE}}
+    CLICKHOUSE_CLUSTER_NAME: {{CLICKHOUSE_CLUSTER_NAME}}
+    DBT_STATE: {{ DBT_STATE_DIR }}
+    ASPECTS_DATA_TTL_EXPRESSION: "{{ ASPECTS_DATA_TTL_EXPRESSION }}"
+    DBT_PROFILE_TARGET_DATABASE: "{{ DBT_PROFILE_TARGET_DATABASE }}"
+    DBT_PROFILES_DIR: {{ DBT_PROFILES_DIR }}
+    DBT_BRANCH: "{{ DBT_BRANCH }}"
+    DBT_REPOSITORY: "{{ DBT_REPOSITORY }}"
+    DBT_SSH_KEY: "{{ DBT_SSH_KEY }}"
   volumes:
     - ../../env/plugins/aspects/apps/aspects:/app/aspects
     - ../../env/plugins/aspects/apps/aspects/scripts/:/app/aspects/scripts:ro
-    - ../../data/aspects-dbt:{{ DBT_STATE_DIR }}
-  {% if RUN_SUPERSET or RUN_CLICKHOUSE or RUN_RALPH %}depends_on:{% if RUN_SUPERSET %}
-    - superset{% endif %}{% if RUN_CLICKHOUSE%}
-    - clickhouse{% endif %}{% if RUN_RALPH %}
-    - ralph{% endif %}
+  {% if RUN_CLICKHOUSE %}
+  depends_on:
+    - clickhouse
   {% endif %}
 clickhouse-job:
     image: {{DOCKER_IMAGE_CLICKHOUSE}}
@@ -26,5 +34,4 @@ superset-job:
   depends_on:
     - mysql
     - redis
-    - superset
 {% endif %}
diff --git a/tutoraspects/patches/local-docker-compose-services b/tutoraspects/patches/local-docker-compose-services
@@ -55,8 +55,6 @@ superset:
   depends_on:
     - mysql
     - redis
-    - superset-worker
-    - superset-worker-beat
 
 superset-worker:
   {% include 'base-docker-compose-services' %}
@@ -123,3 +121,30 @@ aspects-consumer:
     {% if RUN_MONGODB %}- mongodb{% endif %}
     {% if RUN_REDIS %}- redis{% endif %}
 {% endif %}
+
+{% if RUN_ASPECTS_DOCS %}
+aspects-docs:
+  user: root
+  image: {{ DOCKER_IMAGE_ASPECTS }}
+  environment:
+    VENV_DIR: /opt/venv
+    ASPECTS_EVENT_SINK_DATABASE: {{ASPECTS_EVENT_SINK_DATABASE}}
+    ASPECTS_XAPI_DATABASE: {{ASPECTS_XAPI_DATABASE}}
+    CLICKHOUSE_CLUSTER_NAME: {{CLICKHOUSE_CLUSTER_NAME}}
+    DBT_STATE: {{ DBT_STATE_DIR }}
+    ASPECTS_DATA_TTL_EXPRESSION: "{{ ASPECTS_DATA_TTL_EXPRESSION }}"
+    DBT_PROFILE_TARGET_DATABASE: "{{ DBT_PROFILE_TARGET_DATABASE }}"
+    DBT_PROFILES_DIR: {{ DBT_PROFILES_DIR }}
+    DBT_BRANCH: "{{ DBT_BRANCH }}"
+    DBT_REPOSITORY: "{{ DBT_REPOSITORY }}"
+    DBT_SSH_KEY: "{{ DBT_SSH_KEY }}"
+  command: |
+    bash /app/aspects/scripts/docs.sh
+  ports:
+    - 7000:7000
+  volumes:
+    - ../../env/plugins/aspects/apps/aspects/dbt:/app/aspects/dbt
+    - ../../env/plugins/aspects/apps/aspects/scripts/:/app/aspects/scripts:ro
+  depends_on:
+  {% if RUN_CLICKHOUSE %}  - clickhouse{% endif %}
+{% endif %}
diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py
@@ -368,9 +368,12 @@
         ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"),
         ("DBT_BRANCH", "v3.30.0"),
         ("DBT_SSH_KEY", ""),
-        ("DBT_STATE_DIR", "/app/aspects/dbt_state/"),
+        ("DBT_STATE_DIR", "/app/aspects-dbt/state"),
+        ("DBT_PROFILES_DIR", "/app/aspects/dbt/"),
         # This is the name of the database dbt will write to
         ("DBT_PROFILE_TARGET_DATABASE", "reporting"),
+        ("RUN_ASPECTS_DOCS", False),
+        ("DBT_HOST", "dbt.{{LMS_HOST}}"),
     ]
 )
 

diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0038_aspects_data.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0038_aspects_data.py
@@ -0,0 +1,34 @@
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0038"
+down_revision = "0037"
+branch_labels = None
+depends_on = None
+on_cluster = " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " if "{{CLICKHOUSE_CLUSTER_NAME}}" else ""
+engine = (
+    "ReplicatedReplacingMergeTree"
+    if "{{CLICKHOUSE_CLUSTER_NAME}}"
+    else "ReplacingMergeTree"
+)
+
+
+def upgrade():
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.aspects_data
+        {on_cluster}
+        (
+            path String NOT NULL,
+            content String NOT NULL,
+        ) ENGINE {engine}
+        PRIMARY KEY (path);
+        """
+    )
+
+
+def downgrade():
+    op.execute(
+        "DROP TABLE IF EXISTS {{ ASPECTS_EVENT_SINK_DATABASE }}.aspects_data"
+        f"{on_cluster};"
+    )
diff --git a/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh b/tutoraspects/templates/aspects/apps/aspects/scripts/bootstrap.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+
+set -eo pipefail
+
+if [ -z "${DBT_SSH_KEY+x}" ]
+then
+  mkdir -p /root/.ssh
+  echo "${DBT_SSH_KEY}" | tr -d '\r' > /root/.ssh/id_rsa
+  chmod 600 /root/.ssh/id_rsa
+  eval `ssh-agent -s`
+
+  ssh -o StrictHostKeyChecking=no [email protected] || true
+  ssh-add /root/.ssh/id_rsa
+fi
+
+export branch=$(git -C aspects-dbt/ branch --show-current)
+export repo=$(git -C aspects-dbt/ config --get remote.origin.url)
+if [ "$DBT_BRANCH" != "$branch"  ] && [ "$DBT_REPOSITORY" != "$repo" ];
+then
+  rm -rf aspects-dbt
+
+  echo "Installing aspects-dbt"
+  echo "git clone -b ${DBT_BRANCH} ${DBT_REPOSITORY}"
+  git clone -b ${DBT_BRANCH} ${DBT_REPOSITORY} aspects-dbt
+
+  cd aspects-dbt
+
+  if [ -e "./requirements.txt" ]
+  then
+    echo "Installing dbt python requirements"
+    pip install -r ./requirements.txt
+  else
+    echo "No requirements.txt file found; skipping"
+  fi
+
+  echo "Installing dbt dependencies"
+  dbt deps
+
+fi
+
+mkdir -p $DBT_STATE