Merge pull request #2 from battery-pulse/feature/e2e-tests

E2E Tests

.github/workflows/ci.yaml

@@ -0,0 +1,75 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: make install-dev
+
+      - name: Run Formatting Tests
+        run: make test-format
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.11"
+
+      - name: Install Java
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y openjdk-11-jdk
+
+      - name: Install dependencies
+        run: make install-dev
+
+      - name: Run Integration Tests
+        run: make test-integration
+
+  e2e-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.11"
+
+      - name: Install Java
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y openjdk-11-jdk
+
+      - name: Install kind
+        run: |
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/kind
+
+      - name: Install Helm
+        run: |
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+
+      - name: Install dependencies
+        run: make install-dev
+
+      - name: Run E2E Tests
+        run: make test-e2e

.github/workflows/release.yaml

@@ -0,0 +1,54 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read      # Required to access repository content
+      packages: write     # Required for docker image
+      id-token: write     # Required for PyPI
+    steps:
+      - uses: actions/checkout@v2
+
+      # - name: Set up Python
+      #   uses: actions/setup-python@v2
+      #   with:
+      #     python-version: "3.11"
+
+      - name: Get Release Version
+        id: get_version
+        run: echo "package_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+
+      # - name: Build Python Package
+      #   env:
+      #     PACKAGE_VERSION: ${{ env.package_version }}
+      #   run: |
+      #     pip install --upgrade pip
+      #     pip install build twine
+      #     sed -i "s/__version__ = .*/__version__ = \"$PACKAGE_VERSION\"/" src/pulse_telemetry/__init__.py
+      #     python -m build
+
+      # - name: Publish Python Package
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     package_dir: dist
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Docker image
+        run: |
+          docker build -t ghcr.io/battery-pulse/pulse-analytics:${{ env.package_version }} -t ghcr.io/battery-pulse/pulse-analytics:latest .
+
+      - name: Push Docker image
+        run: |
+          docker push ghcr.io/battery-pulse/pulse-analytics:${{ env.package_version }}
+          docker push ghcr.io/battery-pulse/pulse-analytics:latest

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+
+# Install dbt-core and dbt-trino
+RUN pip install --no-cache-dir \
+    dbt-core \
+    dbt-trino
+
+# Set the working directory inside the container
+WORKDIR /dbt
+
+# Copy the dbt project files to the container
+COPY ./dbt /dbt
+
+# Default entrypoint to run dbt build
+ENTRYPOINT ["dbt", "build", "--target", "trino", "--profiles-dir", "."]

Makefile

@@ -26,7 +26,7 @@ test-integration:
 
 .PHONY: test-e2e
 test-e2e:
-	pytest tests/ --dbt-target=trino
+	pytest tests/ -s --log-cli-level=INFO --dbt-target=trino
 
 .PHONY: docker-image
 docker-image:

README.md

@@ -48,7 +48,7 @@ Coming soon...
 
 - **Integration Tests**: Integration tests run on DuckDB to validate data transformations, DBT model execution, and data quality checks. Test setup and teardown is lightweight and fast.
 
-- **End-to-End Tests**: End-to-end tests perform the same checks as integration tests but use Trino as the backend. This setup provides a production-like environment, where telemetry, metadata, and the DBT targets reside in separate data catalogs.
+- **End-to-End Tests**: End-to-end tests perform the same checks as integration tests but use Trino as the backend. This setup provides a production-like environment, where telemetry, metadata, and the DBT targets may reside in separate data catalogs.
 
 ### DBT Project Configuration
 

dbt/macros/prefix_columns.sql

@@ -0,0 +1,8 @@
+{% macro prefix_columns(table, alias) %}
+  {% set columns = adapter.get_columns_in_relation(ref(table)) %}
+  {% set prefixed_columns = [] %}
+  {% for col in columns %}
+    {% do prefixed_columns.append("{}.{} AS {}__{}".format(alias, col.name, alias, col.name)) %}
+  {% endfor %}
+  {{ prefixed_columns | join(', ') }}
+{% endmacro %}

dbt/models/marts/part_statistics_cycle.sql

@@ -8,7 +8,7 @@ WITH test_with_part AS (
     SELECT
         dpt.part_id,
         t.*,  -- All columns from test
-        pm.*  -- All columns from part_metadata
+        {{ prefix_columns('part_metadata', 'pm') }} -- Select all columns from part_metadata
     FROM {{ ref('test_statistics_cycle') }} AS t
     INNER JOIN {{ ref('device_test_part') }} AS dpt
         ON t.device_id = dpt.device_id

dbt/models/marts/part_statistics_step.sql

@@ -8,7 +8,7 @@ WITH test_with_part AS (
     SELECT
         dpt.part_id,
         t.*,  -- All columns from test
-        pm.*  -- All columns from part_metadata
+        {{ prefix_columns('part_metadata', 'pm') }} -- Select all columns from part_metadata
     FROM {{ ref('test_statistics_step') }} AS t
     INNER JOIN {{ ref('device_test_part') }} AS dpt
         ON t.device_id = dpt.device_id

dbt/models/marts/part_telemetry.sql

@@ -8,7 +8,7 @@ WITH test_with_part AS (
     SELECT
         dpt.part_id,
         t.*,  -- All columns from test
-        pm.*  -- All columns from part_metadata
+        {{ prefix_columns('part_metadata', 'pm') }} -- Select all columns from part_metadata
     FROM {{ ref('test_telemetry') }} AS t
     INNER JOIN {{ ref('device_test_part') }} AS dpt
         ON t.device_id = dpt.device_id

dbt/models/marts/test_statistics_cycle.sql

@@ -6,8 +6,8 @@
 SELECT
     dtr.recipe_id,  -- Prefix table with test recipe
     t.*,  -- Select all columns from telemetry
-    dm.*, -- Select all columns from device_metadata
-    rm.*  -- Select all columns from recipe_metadata
+    {{ prefix_columns('device_metadata', 'dm') }}, -- Select all columns from device_metadata
+    {{ prefix_columns('recipe_metadata', 'rm') }}  -- Select all columns from recipe_metadata
 FROM {{ ref('statistics_cycle') }} AS t
 LEFT JOIN {{ ref('device_metadata') }} AS dm  -- Don't drop rows with no device metadata
     ON t.device_id = dm.device_id

dbt/models/marts/test_statistics_step.sql

@@ -6,8 +6,8 @@
 SELECT
     dtr.recipe_id,  -- Prefix table with test recipe
     t.*,  -- Select all columns from telemetry
-    dm.*, -- Select all columns from device_metadata
-    rm.*  -- Select all columns from recipe_metadata
+    {{ prefix_columns('device_metadata', 'dm') }}, -- Select all columns from device_metadata
+    {{ prefix_columns('recipe_metadata', 'rm') }}  -- Select all columns from recipe_metadata
 FROM {{ ref('statistics_step') }} AS t
 LEFT JOIN {{ ref('device_metadata') }} AS dm  -- Don't drop rows with no device metadata
     ON t.device_id = dm.device_id

dbt/models/marts/test_telemetry.sql

@@ -6,8 +6,8 @@
 SELECT
     dtr.recipe_id,  -- Prefix table with test recipe
     t.*,  -- Select all columns from telemetry
-    dm.*, -- Select all columns from device_metadata
-    rm.*  -- Select all columns from recipe_metadata
+    {{ prefix_columns('device_metadata', 'dm') }}, -- Select all columns from device_metadata
+    {{ prefix_columns('recipe_metadata', 'rm') }}  -- Select all columns from recipe_metadata
 FROM {{ ref('telemetry') }} AS t
 LEFT JOIN {{ ref('device_metadata') }} AS dm  -- Don't drop rows with no device metadata
     ON t.device_id = dm.device_id

dbt/profiles.yml

@@ -3,12 +3,16 @@ pulse_analytics:
     trino:
       type: trino
       host: "{{ env_var('PULSE_ANALYTICS_TRINO_HOST') }}"
-      port: "{{ env_var('PULSE_ANALYTICS_TRINO_PORT') }}"
+      port: "{{ env_var('PULSE_ANALYTICS_TRINO_PORT') | int }}"
       user: "{{ env_var('PULSE_ANALYTICS_TRINO_USER') }}"
-      catalog: "{{ env_var('PULSE_ANALYTICS_TARGET_CATALOG') }}"
+      database: "{{ env_var('PULSE_ANALYTICS_TARGET_CATALOG') }}"
       schema: "{{ env_var('PULSE_ANALYTICS_TARGET_SCHEMA') }}"
+      method: ldap
+      password: "{{ env_var('PULSE_ANALYTICS_TRINO_PASSWORD') }}"
+      http_scheme: https
+      cert: false
 
     duckdb:
       type: duckdb
-      path: "{{ env_var('PULSE_ANALYTICS_DUCKDB_PATH') }}"  # Implicitly, the catalog is the file name w/o extension
+      path: "{{ env_var('PULSE_ANALYTICS_DUCKDB_PATH') }}"  # The catalog is the file name w/o extension
       schema: "{{ env_var('PULSE_ANALYTICS_TARGET_SCHEMA') }}"

examples/dbt-job.yaml

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dbt-job
+spec:
+  template:
+    spec:
+      containers:
+      - name: dbt
+        image: pulse-analytics:latest
+        imagePullPolicy: Never
+        envFrom:
+        - configMapRef:
+            name: dbt-config
+        - secretRef:
+            name: trino-users
+        env:
+        - name: PULSE_ANALYTICS_TRINO_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: trino-users
+              key: admin
+      restartPolicy: Never
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dbt-config
+data:
+  # Trino connection
+  PULSE_ANALYTICS_TRINO_HOST: "trino-cluster-coordinator.default.svc.cluster.local"
+  PULSE_ANALYTICS_TRINO_PORT: "8443"
+  PULSE_ANALYTICS_TRINO_USER: "admin"
+
+  # Target catalog
+  PULSE_ANALYTICS_TARGET_CATALOG: "lakehouse"
+  PULSE_ANALYTICS_TARGET_SCHEMA: "analytics"
+
+  # Telemetry sources
+  PULSE_ANALYTICS_TELEMETRY_CATALOG: "lakehouse"
+  PULSE_ANALYTICS_TELEMETRY_SCHEMA: "telemetry"
+  PULSE_ANALYTICS_TELEMETRY_TABLE: "telemetry"
+  PULSE_ANALYTICS_STATISTICS_STEP_TABLE: "statistics_step"
+  PULSE_ANALYTICS_STATISTICS_CYCLE_TABLE: "statistics_cycle"
+
+  # Metadata sources
+  PULSE_ANALYTICS_METADATA_CATALOG: "lakehouse"
+  PULSE_ANALYTICS_METADATA_SCHEMA: "metadata"
+  PULSE_ANALYTICS_DEVICE_METADATA_TABLE: "device_metadata"
+  PULSE_ANALYTICS_RECIPE_METADATA_TABLE: "recipe_metadata"
+  PULSE_ANALYTICS_DEVICE_TEST_RECIPE_TABLE: "device_test_recipe"
+  PULSE_ANALYTICS_PART_METADATA_TABLE: "part_metadata"
+  PULSE_ANALYTICS_DEVICE_TEST_PART_TABLE: "device_test_part"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-users
+type: kubernetes.io/opaque
+stringData:
+  admin: admin

pyproject.toml

@@ -20,6 +20,8 @@ dev = [
     "pytest==8.2.2",
     "ruff==0.5.2",
     "setuptools==75.1.0",
+    "sqlalchemy==2.0.36",
+    "sqlalchemy-trino==0.5.0",
     "pulse-telemetry==0.1.1",
 ]