Merge pull request #1 from gnosischain/dbt_configs

Configs

.env.test

@@ -0,0 +1,5 @@
+CLICKHOUSE_URL
+CLICKHOUSE_PORT
+CLICKHOUSE_USER
+CLICKHOUSE_PASSWORD
+CLICKHOUSE_SECURE

Dockerfile

@@ -7,15 +7,28 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Create a non-root user
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+RUN addgroup --gid $GROUP_ID appgroup && \
+    adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID appuser
+
 # Setup the working directory
 WORKDIR /app
+RUN chown -R appuser:appgroup /app && chmod -R 755 /app
+
+# Switch to non-root user
+USER appuser
 
 # Copy the Python requirements and install them
-COPY /requirements.txt /app/requirements.txt
-RUN pip install -r /app/requirements.txt
+COPY requirements.txt /app/requirements.txt
+RUN pip install --user -r /app/requirements.txt
 
 # Set environment variable to specify the DBT project path
 ENV DBT_PROJECT_PATH /app/src
 
 # Optionally expose a port for dbt docs if needed
-EXPOSE 8080
+EXPOSE 8080
+
+# Set PATH to include user-level binaries
+ENV PATH=/home/appuser/.local/bin:$PATH

docker-compose.yml

@@ -0,0 +1,34 @@
+services:
+  dbt:
+    container_name: dbt
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        USER_ID: ${USER_ID:-1000}
+        GROUP_ID: ${GROUP_ID:-1000}
+    restart: unless-stopped
+    ports:
+      - "8080:8000"
+    command: > 
+      /bin/bash -c "exec python -m http.server 8000 --directory logs & tail -f /dev/null"
+    environment:
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL}
+      CLICKHOUSE_PORT: ${CLICKHOUSE_PORT}
+      CLICKHOUSE_SECURE: ${CLICKHOUSE_SECURE}
+    volumes:
+      - type: bind
+        source: ./
+        target: /app
+        consistency: cached
+      - ./profiles.yml:/home/appuser/.dbt/profiles.yml
+    working_dir: /app
+    user: appuser
+    networks:
+      - dbt_net
+
+
+networks:
+  dbt_net:

models/transformations/power_consumption/consensus_power.sql

@@ -1,22 +1,14 @@
 WITH consensus_power AS (
-    SELECT * FROM (
-        VALUES 
-            (4, 'Lighthouse', 2.75),
-            (5, 'Lighthouse', 3.14),
-            (6, 'Lighthouse', 18.84),
-            (4, 'Teku', 3.71),
-            (5, 'Teku', 3.32),
-            (6, 'Teku', 27.46),
-            (4, 'Lodestar', 3.14),
-            (5, 'Lodestar', 3.89),
-            (6, 'Lodestar', 33.55),
-            (4, 'Nimbus', 1.67),
-            (5, 'Nimbus', 2.08),
-            (6, 'Nimbus', 17.11),
-            (4, 'Prysm', 3.51),
-            (5, 'Prysm', 2.87),
-            (6, 'Prysm', 24.33)
-    ) AS t(type, client, mean)
+    SELECT
+        type,
+        client,
+        mean
+    FROM (
+        SELECT
+            arrayJoin([4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6]) AS type,
+            arrayJoin(['Lighthouse', 'Lighthouse', 'Lighthouse', 'Teku', 'Teku', 'Teku', 'Lodestar', 'Lodestar', 'Lodestar', 'Nimbus', 'Nimbus', 'Nimbus', 'Prysm', 'Prysm', 'Prysm']) AS client,
+            arrayJoin([2.75, 3.14, 18.84, 3.71, 3.32, 27.46, 3.14, 3.89, 33.55, 1.67, 2.08, 17.11, 3.51, 2.87, 24.33]) AS mean
+    )
 )
 
 SELECT * FROM consensus_power

profiles.yml

@@ -1,17 +1,41 @@
 gnosis_dbt:
-  target: clickhouse
+  target: ch-dbt
   outputs:
-    clickhouse:  # ClickHouse configuration for additional data sourcing
+    ch-dbt:
       type: clickhouse
-      schema: "{{ env_var('CLICKHOUSE_SCHEMA') }}"
+      schema: "dbt"
       verify: False
       host: "{{ env_var('CLICKHOUSE_URL') }}"
       port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}"
       secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}"
-      cluster: '{cluster}'
-      cluster_mode: True
       user: "{{ env_var('CLICKHOUSE_USER') }}"
       password: "{{ env_var('CLICKHOUSE_PASSWORD') }}"
       threads: 40
       connect_timeout: 60  # Increase the connect timeout (in seconds)
       read_timeout: 3000    # Increase the read timeout (in seconds)
+
+    ch-valtrack: 
+      type: clickhouse
+      schema: "valtrack_preview"
+      verify: False
+      host: "{{ env_var('CLICKHOUSE_URL') }}"
+      port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}"
+      secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}"
+      user: "{{ env_var('CLICKHOUSE_USER') }}"
+      password: "{{ env_var('CLICKHOUSE_PASSWORD') }}"
+      threads: 40
+      connect_timeout: 60
+      read_timeout: 3000
+
+    ch-goteth: 
+      type: clickhouse
+      schema: "goteth_preview"
+      verify: False
+      host: "{{ env_var('CLICKHOUSE_URL') }}"
+      port: "{{ env_var('CLICKHOUSE_PORT', '8123') | int }}"
+      secure: "{{ (env_var('CLICKHOUSE_SECURE', 'False') | lower) == 'true' }}"
+      user: "{{ env_var('CLICKHOUSE_USER') }}"
+      password: "{{ env_var('CLICKHOUSE_PASSWORD') }}"
+      threads: 40
+      connect_timeout: 60
+      read_timeout: 3000

requirements.txt

@@ -0,0 +1,3 @@
+dbt-core==1.8.7
+dbt-clickhouse==1.8.4
+pytest==8.2.1