Run E2E Test w/ Github Actions

.dockerignore

@@ -73,3 +73,7 @@ distribute-*.tar.gz
 *.xml
 
 *.iml
+
+.venv/
+
+.devenv/

.github/workflows/e2e.yaml

@@ -12,41 +12,56 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install kubectl
-        uses: azure/setup-kubectl@v3
+      - name: Install nix
+        uses: cachix/install-nix-action@v24
         with:
-          version: "v1.22.17"
+          extra_nix_config: |
+            accept-flake-config = true
 
-      - name: Start a Kind cluster
-        uses: helm/kind-action@v1.8.0
+      - name: Setup cachix
+        uses: cachix/cachix-action@v12
         with:
-          cluster_name: kind
-          version: "v0.19.0"
-          node_image: kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5
-          wait: "600s"
+          name: lco-public
+          extraPullNames: devenv
+          skipPush: true
 
-      - name: Build docker image
-        run: |
-          docker build -t banzai:test-latest .
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v3
 
-      - name: Copy docker image to nodes
+      - name: Start Kind cluster
+        shell: nix develop --impure --command bash -e {0}
         run: |
-          kind load docker-image banzai:test-latest
+          kind-create-cluster
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: "network=host"
 
-      - name: Start banzai
+      - name: Deploy Banzai w/ Skaffold
+        env:
+          SKAFFOLD_BUILDX_ARGS: "--cache-to type=gha,mode=max,ignore-error=true --cache-from type=gha"
+        shell: nix develop --impure --command bash -e {0}
         run: |
-          cat banzai/tests/e2e-k8s.yaml
+          docker ps
+          skaffold -p e2e run
 
-          # Deploy banzai stack
-          kubectl apply -f banzai/tests/e2e-k8s.yaml
+      - name: Test Super Bias Creation
+        shell: nix develop --impure --command bash -e {0}
+        run: |
+          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
 
-          # Wait for banzai to be ready
-          kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test
+      - name: Test Super Dark Creation
+        shell: nix develop --impure --command bash -e {0}
+        run: |
+          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark
 
-      - name: Test Master Bias Creation
+      - name: Test Super Flat Creation
+        shell: nix develop --impure --command bash -e {0}
         run: |
-          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
-
-      - name: Cleanup
+          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat
+
+      - name: Test Science File Creation
+        shell: nix develop --impure --command bash -e {0}
         run: |
-          kubectl delete pod banzai-e2e-test
+          kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files

.gitignore

@@ -75,3 +75,5 @@ distribute-*.tar.gz
 *.iml
 
 .coverage.subprocess
+
+.devenv/

banzai/main.py

@@ -22,9 +22,9 @@
 from banzai.utils import date_utils, stage_utils, import_utils, image_utils, fits_utils, file_utils
 from banzai.celery import process_image, app, schedule_calibration_stacking
 from banzai.data import DataProduct
+from celery.apps.beat import Beat
 from celery.schedules import crontab
 import celery
-import celery.bin.beat
 import requests
 
 logger = logs.get_logger()
@@ -174,9 +174,9 @@ def start_stacking_scheduler():
                               schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)),
                               queue=runtime_context.CELERY_TASK_QUEUE_NAME)
 
-    beat = celery.bin.beat.beat(app=app)
+    beat = Beat(app=app, loglevel='INFO', schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid')
     logger.info('Starting celery beat')
-    beat.run(schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid', working_directory='/tmp')
+    beat.run()
 
 
 def run_realtime_pipeline():