fix: Fix dependencies and the login

.env.tpl

@@ -11,7 +11,7 @@ AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_C
 AIRFLOW__CORE__FERNET_KEY=${AIRFLOW__CORE__FERNET_KEY}
 AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
 AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW__WEBSERVER__SECRET_KEY}
-AIRFLOW__SMTP__SMTP_HOST=${AIRFLOW__SMTP__SMTP_HOST"}
+AIRFLOW__SMTP__SMTP_HOST=${AIRFLOW__SMTP__SMTP_HOST}
 AIRFLOW__SMTP__SMTP_STARTTLS=${AIRFLOW__SMTP__SMTP_STARTTLS:-False}
 AIRFLOW__SMTP__SMTP_SSL=${AIRFLOW__SMTP__SMTP_SSL:-False}
 AIRFLOW__SMTP__SMTP_USER=${AIRFLOW__SMTP__SMTP_USER}

.github/workflows/main.yaml

@@ -79,7 +79,7 @@ jobs:
       cancel-in-progress: true
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Create environment variables file
       run: |
@@ -90,39 +90,37 @@ jobs:
         envsubst < .env.tpl > .env
         mkdir /home/runner/work/psql_data
 
-    - name: setup
-      run: |
-        make prepare-host
-
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
-        mamba-version: "*"
         environment-file: conda/base.yaml
         channels: conda-forge,r,nodefaults
         activate-environment: epigraphhub
-        use-mamba: true
-        miniforge-variant: Mambaforge
+        auto-update-conda: true
+        conda-solver: libmamba
+
+    - name: setup
+      run: makim host.prepare
 
     - name: configure epigraphhub credentials
       run: |
         epigraphhub-config --name ci-epigraphhub \
           --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}"
 
     - name: configure aws s3 credentials to minio
-      run: make dev-create-s3-credential
+      run: makim aws.create-s3-credential
 
     - name: build docker containers
-      run: make containers-build
+      run: sugar build
 
     - name: start services
       run: |
-        make containers-start-services SERVICES="superset airflow postgres"      
+        sugar ext restart --services superset,airflow,postgres --options -d
 
     - name: wait for the services are properly working
       run: |
-        docker ps
-        make containers-wait-all
+        sugar ps
+        makim containers.wait-all
 
     - name: run epigraphhub tests
       run: |
@@ -146,7 +144,7 @@ jobs:
     - name: Generate logs
       if: ${{ failure() }}
       run: |
-        make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log
+        sugar logs --options --tail 1000 > /tmp/containers-services.log
 
     - name: Archive log artifacts
       uses: actions/upload-artifact@v3

.makim.yaml

@@ -0,0 +1,130 @@
+version: 1.0
+env-file: .env
+shell: bash
+
+groups:
+  host:
+    tasks:
+      prepare:
+        help: Prepare host environment
+        run: bash scripts/prepare-host.sh
+
+  containers:
+    tasks:
+      pull:
+        help: Pull container images
+        run: |
+          set -e
+          docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml pull ${SERVICES}
+
+      build:
+        help: Build containers
+        run: |
+          set -e
+          docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml build ${SERVICES}
+
+      start:
+        help: Start containers
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml up --remove-orphans -d ${SERVICES}
+
+      stop:
+        help: Stop containers
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml stop ${ARGS} ${SERVICES}
+
+      remove:
+        help: Remove containers
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml rm ${ARGS} ${SERVICES}
+
+      restart:
+        help: Restart containers
+        dependencies:
+          - task: containers.stop
+          - task: containers.start
+
+      logs:
+        help: Show container logs
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml logs ${ARGS} ${SERVICES}
+
+      logs-follow:
+        help: Follow container logs
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml logs --follow ${ARGS} ${SERVICES}
+
+      exec:
+        help: Execute command in a running container
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml exec ${ARGS} ${SERVICE} ${CMD}
+
+      console:
+        help: Open console in a container
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml exec ${ARGS} ${SERVICE} bash
+
+      down:
+        help: Take down containers and volumes
+        run: docker-compose --env-file=.env --project-name egh-${ENV} --file containers/compose-base.yaml --file containers/compose-${ENV}.yaml down --volumes --remove-orphans
+
+      reset-storage:
+        help: Reset container storage
+        run: rm -rf ~/.local/share/containers/
+
+      wait:
+        help: Wait until container is healthy
+        args:
+          service:
+            help: the name of the service
+            type: str
+            required: true
+          timeout:
+            help: the time (in seconds) for waiting the service
+            type: int
+            default: 90
+            required: false
+        run: |
+          timeout ${{ args.timeout }} ./containers/healthcheck.sh ${{ args.service }}
+
+      wait-all:
+        help: Wait until container is healthy
+        args:
+          timeout:
+            help: the time (in seconds) for waiting the service
+            type: int
+            default: 90
+            required: false
+        dependencies:
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: postgres
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: minio
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: redis
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: flower
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: superset
+          - target: wait
+            args:
+              timeout: ${{ args.timeout }}
+              service: airflow
+
+  aws:
+    tasks:
+      create-s3-credential:
+        help: Create S3 credentials for development
+        run: ./scripts/dev/create-s3-credential.sh
+
+  conda:
+    tasks:
+      lock:
+        help: Generate lock files for Conda environments
+        run: |
+          cd conda
+          rm -f conda-*.lock
+          conda-lock --conda `which mamba` -f prod.yaml -p osx-64 -p linux-64 --kind explicit

.sugar.yaml

@@ -0,0 +1,50 @@
+version: 1.0.0
+compose-app: docker-compose
+env-file: .env
+defaults:
+  group: {{ env.ENV }}
+
+groups:
+  airflow:
+    project-name: egh-airflow
+    compose-path:
+      - containers/compose-airflow.yaml
+    env-file: containers/airflow/.env
+    services:
+      default: webserver,scheduler,worker,triggerer
+      available:
+        - name: webserver
+        - name: scheduler
+        - name: worker
+        - name: triggerer
+        - name: airflow-cli
+
+  dev:
+    project-name: egh-dev
+    compose-path:
+      - containers/compose-base.yaml
+      - containers/compose-dev.yaml
+    services:
+      default: superset,airflow
+      available:
+        - name: superset
+        - name: airflow
+        - name: postgres
+        - name: minio
+        - name: redis
+        - name: flower
+
+  prod:
+    project-name: egh-prod
+    compose-path:
+      - containers/compose-base.yaml
+      - containers/compose-prod.yaml
+    services:
+      default: superset,airflow
+      available:
+        - name: superset
+        - name: airflow
+        - name: postgres
+        - name: minio
+        - name: redis
+        - name: flower

conda/base.yaml

@@ -11,7 +11,8 @@ dependencies:
   - webdriver-manager
   - pip
   - pip:
-    - containers-sugar
+    - containers-sugar==1.11.4
+    - makim==1.15.2
     - compose-go
-    - epigraphhub
+    - epigraphhub>=2.1.1
     - "selenium<=4.0"

containers/airflow/envs/epigraphhub.txt

@@ -1 +1 @@
-epigraphhub >= 2.1.0
+epigraphhub >= 2.1.1

containers/superset/superset_config.py

@@ -10,6 +10,9 @@
 TALISMAN_ENABLED = None
 TALISMAN_CONFIG = {}
 
+# github.com/apache/superset/discussions/24435#discussioncomment-6282883
+# SESSION_PROTECTION = "basic"
+
 SESSION_COOKIE_SAMESITE = "Lax"
 # Flask-WTF flag for CSRF
 WTF_CSRF_ENABLED = os.getenv("ENV") == "prod"
@@ -110,7 +113,7 @@ class CeleryConfig:
     DB_USER = os.getenv("SUPERSET_DB_USER")
     DB_PASS = os.getenv("SUPERSET_DB_PASSWORD")
     DB_DATABASE = os.getenv("SUPERSET_DB_DATABASE")
-else: 
+else:
     DB_USER = "dev_admin"
     DB_PASS = "admin"
     DB_DATABASE = "dev_superset"