Merge pull request #1 from GovA11y/migrations

Auto-Migrate

Dockerfile

@@ -8,17 +8,20 @@ COPY startup/ /docker-entrypoint-initdb.d/
 # Copy configuration files directly into /etc/clickhouse-server/
 COPY config/ /etc/clickhouse-server/
 
+# Copy & Setup Migrations
+COPY migrations/ /etc/clickhouse-server/migrations/
+
+
+# Ensure init.sh is executable
+RUN chmod +x /docker-entrypoint-initdb.d/init.sh
+
+
 # Define a volume for the data
 VOLUME /var/lib/clickhouse
 
 # Define a volume for the logs
 VOLUME /var/log/clickhouse-server
 
-# Set environment variables
-ENV CLICKHOUSE_DB=gova11y
-ENV CLICKHOUSE_USER=a11ypython
-ENV CLICKHOUSE_PASSWORD=SnakeInTheData
-
 # Do some housekeeping
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8

migrations/001.sql

@@ -0,0 +1,26 @@
+-- Migrations: 001
+-- Create axe_tests table
+CREATE TABLE gova11y.axe_tests
+(
+    `domain_id` Int32,
+    `domain` String,
+    `url_id` Int64,
+    `url` String,
+    `scan_id` Int64,
+    `rule_id` Int64,
+    `test_id` UUID DEFAULT generateUUIDv4(),
+    `tested_at` DateTime,
+    `rule_type` String,
+    `axe_id` String,
+    `impact` String,
+    `target` String,
+    `html` String,
+    `failure_summary` Nullable(String),
+    `created_at` DateTime DEFAULT now(),
+    `active` UInt8 DEFAULT 1,
+    `section508` UInt8 DEFAULT 0,
+    `super_waggy` UInt8 DEFAULT 0
+)
+ENGINE = MergeTree
+ORDER BY test_id
+SETTINGS index_granularity = 8192;

startup/init.sql → migrations/002.sql

@@ -1,32 +1,4 @@
--- GovA11y ClickHouse Setup Script
-
-
--- Create axe_tests table
-CREATE TABLE gova11y.axe_tests
-(
-    `domain_id` Int32,
-    `domain` String,
-    `url_id` Int64,
-    `url` String,
-    `scan_id` Int64,
-    `rule_id` Int64,
-    `test_id` UUID DEFAULT generateUUIDv4(),
-    `tested_at` DateTime,
-    `rule_type` String,
-    `axe_id` String,
-    `impact` String,
-    `target` String,
-    `html` String,
-    `failure_summary` Nullable(String),
-    `created_at` DateTime DEFAULT now(),
-    `active` UInt8 DEFAULT 1,
-    `section508` UInt8 DEFAULT 0,
-    `super_waggy` UInt8 DEFAULT 0
-)
-ENGINE = MergeTree
-ORDER BY test_id
-SETTINGS index_granularity = 8192;
-
+-- Migrations: 002
 -- Create Recent Tests Table
 -- gova11y.axe_tests_recent source
 
@@ -103,25 +75,3 @@ FROM
     FROM gova11y.axe_tests
 ) AS subquery
 WHERE row_num = 1;
-
--- Create Recent Tests Score Table
--- gova11y.axe_tests_recent_score definition
-
-CREATE TABLE gova11y.axe_tests_recent_score
-(
-    `domain` String,
-    `url` String,
-    `axe_id` String,
-    `rule_type` String,
-    `impact` String,
-    `count_tests` Int64,
-    `created_at` DateTime DEFAULT now(),
-    `updated_at` DateTime DEFAULT now()
-)
-ENGINE = MergeTree
-ORDER BY (domain,
- url,
- axe_id,
- rule_type,
- impact)
-SETTINGS index_granularity = 8192;

migrations/003.sql

@@ -0,0 +1,22 @@
+-- Migrations: 003
+-- Create Recent Tests Score Table
+-- gova11y.axe_tests_recent_score definition
+
+CREATE TABLE gova11y.axe_tests_recent_score
+(
+    `domain` String,
+    `url` String,
+    `axe_id` String,
+    `rule_type` String,
+    `impact` String,
+    `count_tests` Int64,
+    `created_at` DateTime DEFAULT now(),
+    `updated_at` DateTime DEFAULT now()
+)
+ENGINE = MergeTree
+ORDER BY (domain,
+ url,
+ axe_id,
+ rule_type,
+ impact)
+SETTINGS index_granularity = 8192;

startup/init.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# ClickHouse Migration Script
+# startup/init.sh
+
+# Directory containing migration scripts
+MIGRATIONS_DIR="/etc/clickhouse-server/migrations"
+
+# Check if the migrations table exists, and if not, create it
+if ! clickhouse-client --query "EXISTS TABLE gova11y.migrations"; then
+    clickhouse-client --query "
+        CREATE TABLE gova11y.migrations
+        (
+            migration_name String,
+            applied_at DateTime DEFAULT now()
+        )
+        ENGINE = MergeTree
+        ORDER BY migration_name
+        SETTINGS index_granularity = 8192;"
+fi
+
+# Loop through migration scripts in order
+for migration in $(ls $MIGRATIONS_DIR | sort); do
+    # Check if the migration has been applied already
+    if ! clickhouse-client --query "SELECT 1 FROM gova11y.migrations WHERE migration_name='$migration'"; then
+        clickhouse-client < "$MIGRATIONS_DIR/$migration"
+        if [ $? -ne 0 ]; then
+            # If there's an error running the script, exit
+            echo "Error applying migration $migration"
+            exit 1
+        else
+            echo "Applied migration $migration"
+            # Record the migration in the migrations table
+            clickhouse-client --query "INSERT INTO gova11y.migrations (migration_name) VALUES ('$migration')"
+        fi
+    else
+        echo "Migration $migration already applied, skipping"
+    fi
+done
+
+