Superset deployment

pyproject.toml

@@ -152,6 +152,7 @@ dev = [
     "ruff-lsp",
     "jupyter-resource-usage",
     "pygraphviz",
+    "terraform>=1.9.2"
 ]
 
 [tool.setuptools]

superset/Dockerfile

@@ -0,0 +1,14 @@
+FROM apache/superset:4.0.2
+
+# hadolint ignore=DL3002
+USER root
+
+COPY --chown=superset superset_config.py /app/
+ENV SUPERSET_CONFIG_PATH /app/superset_config.py
+
+# add to requirements file
+COPY --chown=superset requirements.txt /app/
+RUN pip install --no-cache-dir -r /app/requirements.txt && \
+    rm /app/requirements.txt
+# Switching back to using the `superset` user
+USER superset

superset/README.md

@@ -0,0 +1,49 @@
+# PUDL Superset
+This directory contains files required to build and deploy PUDL's Superset instance.
+
+## Local deployment
+To test out a local deployment build the images:
+
+```
+docker compose build
+```
+
+Before you start the service you'll need to set some environment variables
+
+```
+# These auth0 are required for authentication
+# For local development it's best that you create your own
+# Auth0 project so we don't accidently muck with the production
+# auth0 information.
+# You can follow the instructions here: https://auth0.com/docs/get-started/auth0-overview/create-applications
+export AUTH0_CLIENT_ID="auth0 client id"
+export AUTH0_CLIENT_SECRET="auth0 client secret"
+export AUTH0_DOMAIN="auth0 client domain"
+
+# Set the connection details
+export SUPERSET_DB_HOST=postgres
+export SUPERSET_DB_PORT=5432
+export SUPERSET_DB_USER=superset
+export SUPERSET_DB_PASS=superset
+export SUPERSET_DB_NAME=superset
+```
+
+Then you can start the services
+
+```
+docker compose up
+```
+
+If this is the first time running superset locally or you recently ran `docker compose down` you'll need to run the commands in `setup.sh`.
+
+## Making changes to the production deployment
+TODO: instructions on how to connect to Cloud SQL
+
+## Deploy to Cloud Run
+Once you've made changes to the superset docker image, you can update the production deployment with this command:
+
+```
+gcloud builds submit --config cloudbuild.yaml .
+```
+
+This command will use Cloud Build to build the docker image, push it to the Google Cloud Artifact Registry and redeploy the Cloud Run `pudl-superset` service with the new docker image.

superset/cloudbuild.yaml

@@ -0,0 +1,29 @@
+steps:
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      [
+        "build",
+        "-t",
+        "us-central1-docker.pkg.dev/catalyst-cooperative-pudl/pudl-superset/pudl-superset:latest",
+        "--platform",
+        "linux/amd64",
+        ".",
+      ]
+  - name: "gcr.io/cloud-builders/docker"
+    args:
+      [
+        "push",
+        "us-central1-docker.pkg.dev/catalyst-cooperative-pudl/pudl-superset/pudl-superset:latest",
+      ]
+
+  - name: "gcr.io/cloud-builders/gcloud"
+    args:
+      [
+        "run",
+        "deploy",
+        "pudl-superset",
+        "--image",
+        "us-central1-docker.pkg.dev/catalyst-cooperative-pudl/pudl-superset/pudl-superset:latest",
+        "--region",
+        "us-central1",
+      ]

superset/docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  superset:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: pudl-superset
+    environment:
+      AUTH0_CLIENT_ID: ${AUTH0_CLIENT_ID}
+      AUTH0_CLIENT_SECRET: ${AUTH0_CLIENT_SECRET}
+      AUTH0_DOMAIN: ${AUTH0_DOMAIN}
+      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
+      SUPERSET_DB_HOST: ${SUPERSET_DB_HOST-postgres}
+      SUPERSET_DB_PORT: ${SUPERSET_DB_PORT-5432}
+      SUPERSET_DB_USER: ${SUPERSET_DB_USER-superset}
+      SUPERSET_DB_PASS: ${SUPERSET_DB_PASS-superset}
+      SUPERSET_DB_NAME: ${SUPERSET_DB_NAME-superset}
+    ports:
+      - "8080:8088"
+    volumes:
+      - ${PUDL_OUTPUT}/pudl.duckdb:/app/pudl.duckdb
+      - ./roles.json:/app/roles.json
+    depends_on:
+      - postgres
+  postgres:
+    image: postgis/postgis:13-3.1-alpine
+    environment:
+      POSTGRES_DB: superset
+      POSTGRES_USER: superset
+      POSTGRES_PASSWORD: superset
+    ports:
+      - 8081:5432

superset/requirements.txt

@@ -0,0 +1,5 @@
+duckdb==1.0.0
+duckdb-engine==0.13.0
+psycopg2-binary==2.9.9
+Authlib==1.3.1
+pg8000==1.31.2

superset/setup.sh

@@ -0,0 +1,16 @@
+# Description: This script was used to setup the superset instance for the first time.
+
+# Create and admin user
+docker compose exec -it superset superset fab create-admin \
+              --username admin \
+              --firstname Superset \
+              --lastname Admin \
+              --email [email protected] \
+              --password admin
+
+# Initialize the database and run migrations
+docker compose exec -it superset superset db upgrade
+docker compose exec -it superset superset init
+
+# Import custom roles that include a new role that combines permissions of Gamma and sql_user roles
+# docker exec -it pudl-superset superset fab import-roles --path /app/roles.json

superset/superset_config.py

@@ -0,0 +1,74 @@
+"""PUDL's Superset configuration."""
+
+import os
+
+import sqlalchemy as sa
+from flask import Flask
+from flask_appbuilder.security.manager import (
+    AUTH_OAUTH,
+)
+
+AUTH_TYPE = AUTH_OAUTH
+
+AUTH0_CLIENT_ID = os.environ["AUTH0_CLIENT_ID"]
+AUTH0_CLIENT_SECRET = os.environ["AUTH0_CLIENT_SECRET"]
+AUTH0_DOMAIN = os.environ["AUTH0_DOMAIN"]
+
+OAUTH_PROVIDERS = [
+    {
+        "name": "auth0",
+        "token_key": "access_token",  # Name of the token in the response of access_token_url
+        "icon": "fa-key",  # Icon for the provider
+        "remote_app": {
+            "client_id": AUTH0_CLIENT_ID,  # Client Id (Identify Superset application)
+            "client_secret": AUTH0_CLIENT_SECRET,  # Secret for this Client Id (Identify Superset application)
+            "client_kwargs": {"scope": "openid profile email groups"},
+            "server_metadata_url": f"https://{AUTH0_DOMAIN}/.well-known/openid-configuration",
+        },
+    }
+]
+
+AUTH_USER_REGISTRATION = True
+AUTH_USER_REGISTRATION_ROLE = "GammaSQLLab"
+
+
+def get_db_connection_string() -> str:
+    """Get the database connection string."""
+    drivername = "postgresql+psycopg2"
+    host = os.environ.get("SUPERSET_DB_HOST")
+    port = os.environ.get("SUPERSET_DB_PORT")
+    username = os.environ["SUPERSET_DB_USER"]
+    password = os.environ["SUPERSET_DB_PASS"]
+    database = os.environ["SUPERSET_DB_NAME"]
+
+    is_cloud_run = os.environ.get("IS_CLOUD_RUN", False)
+
+    if is_cloud_run:
+        cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME")
+        # I couldn't figure out how to use unix sockets with the sa.engine.url.URL
+        # class so I'm creating the connection string manually
+        return f"postgresql+psycopg2://{username}:{password}@/{database}?host=/cloudsql/{cloud_sql_connection_name}"
+    return str(
+        sa.engine.url.URL.create(
+            drivername=drivername,
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            database=database,
+        )
+    )
+
+
+SQLALCHEMY_DATABASE_URI = get_db_connection_string()
+
+
+def FLASK_APP_MUTATOR(app: Flask) -> None:  # noqa: N802
+    """Superset function that allows you to configure the Flask app.
+
+    Args:
+        app: The Flask app instance
+    """
+    app.config.update(
+        PREFERRED_URL_SCHEME="https",
+    )