diff --git a/.dockerignore b/.dockerignore index 85f331d9..a65e23a2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,3 +73,7 @@ distribute-*.tar.gz *.xml *.iml + +.venv/ + +.devenv/ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d43efe84..382a3dcb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,41 +12,56 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install kubectl - uses: azure/setup-kubectl@v3 + - name: Install nix + uses: cachix/install-nix-action@v24 with: - version: "v1.22.17" + extra_nix_config: | + accept-flake-config = true - - name: Start a Kind cluster - uses: helm/kind-action@v1.8.0 + - name: Setup cachix + uses: cachix/cachix-action@v12 with: - cluster_name: kind - version: "v0.19.0" - node_image: kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5 - wait: "600s" + name: lco-public + extraPullNames: devenv + skipPush: true - - name: Build docker image - run: | - docker build -t banzai:test-latest . + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 - - name: Copy docker image to nodes + - name: Start Kind cluster + shell: nix develop --impure --command bash -e {0} run: | - kind load docker-image banzai:test-latest + kind-create-cluster + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: "network=host" - - name: Start banzai + - name: Deploy Banzai w/ Skaffold + env: + SKAFFOLD_BUILDX_ARGS: "--cache-to type=gha,mode=max,ignore-error=true --cache-from type=gha" + shell: nix develop --impure --command bash -e {0} run: | - cat banzai/tests/e2e-k8s.yaml + docker ps + skaffold -p e2e run - # Deploy banzai stack - kubectl apply -f banzai/tests/e2e-k8s.yaml + - name: Test Super Bias Creation + shell: nix develop --impure --command bash -e {0} + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias - # Wait for banzai to be ready - kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test + - name: Test Super Dark Creation + shell: nix develop --impure --command bash -e {0} + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark - - name: Test Master Bias Creation + - name: Test Super Flat Creation + shell: nix develop --impure --command bash -e {0} run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias - - - name: Cleanup + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + + - name: Test Science File Creation + shell: nix develop --impure --command bash -e {0} run: | - kubectl delete pod banzai-e2e-test + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files diff --git a/.gitignore b/.gitignore index acb9875b..168e0832 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,5 @@ distribute-*.tar.gz *.iml .coverage.subprocess + +.devenv/ diff --git a/banzai/main.py b/banzai/main.py index e0fc4e66..9a45430d 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -22,9 +22,9 @@ from banzai.utils import date_utils, stage_utils, import_utils, image_utils, fits_utils, file_utils from banzai.celery import process_image, app, schedule_calibration_stacking from banzai.data import DataProduct +from celery.apps.beat import Beat from celery.schedules import crontab import celery -import celery.bin.beat import requests logger = logs.get_logger() @@ -174,9 +174,9 @@ def start_stacking_scheduler(): schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)), queue=runtime_context.CELERY_TASK_QUEUE_NAME) - beat = celery.bin.beat.beat(app=app) + beat = Beat(app=app, loglevel='INFO', schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid') logger.info('Starting celery beat') - beat.run(schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid', working_directory='/tmp') + beat.run() def run_realtime_pipeline(): diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 53180e25..b3fc11e9 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -14,224 +14,234 @@ spec: fsGroup: 10000 containers: - - name: banzai-redis - image: redis:5.0.3 - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 0.1 - memory: 256Mi - limits: - cpu: 1 - memory: 512Mi - readinessProbe: - exec: - command: - - redis-cli - - ping - - | - - grep - - "PONG" - initialDelaySeconds: 5 - periodSeconds: 1 - - name: banzai-fits-exchange - image: rabbitmq:3.12.3 - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 1 - memory: 512Mi - limits: - cpu: 2 - memory: 512Mi - readinessProbe: - exec: - command: - - rabbitmqctl - - node_health_check - initialDelaySeconds: 5 - periodSeconds: 1 - timeoutSeconds: 10 - - name: banzai-celery-workers - image: banzai:test-latest - imagePullPolicy: IfNotPresent - volumeMounts: - - name: banzai-data - mountPath: /archive/engineering - subPath: engineering - readOnly: false - env: - - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" - - name: RETRY_DELAY - value: "0" - - name: TASK_HOST - value: "redis://localhost:6379/0" - - name: BANZAI_WORKER_LOGLEVEL - value: debug - - name: CALIBRATE_PROPOSAL_ID - value: "calibrate" - - name: OBSERVATION_PORTAL_URL - value: "http://internal-observation-portal.lco.gtn/api/observations/" - - name: API_ROOT - value: "https://archive-api.lco.global/" - - name: OMP_NUM_THREADS - value: "2" - - name: FITS_EXCHANGE - value: "fits_files" - - name: OPENTSDB_PYTHON_METRICS_TEST_MODE - value: "1" - - name: CELERY_TASK_QUEUE_NAME - value: "e2e_task_queue" - - name: REFERENCE_CATALOG_URL - value: "http://phot-catalog.lco.gtn/" - command: - - celery - - -A - - banzai - - worker - - --hostname - - "banzai-celery-worker" - - --concurrency - - "2" - - -l - - "info" - - "-Q" - - "$(CELERY_TASK_QUEUE_NAME)" - - "-n" - - "celery-worker" - - readinessProbe: - exec: - command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" - initialDelaySeconds: 5 - periodSeconds: 1 - timeoutSeconds: 10 - resources: - requests: - cpu: 4 - memory: 4Gi - limits: - cpu: 8 - memory: 5Gi - - name: banzai-large-celery-workers - image: banzai:test-latest - imagePullPolicy: IfNotPresent - volumeMounts: - - name: banzai-data - mountPath: /archive/engineering - subPath: engineering - readOnly: false - env: - - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" - - name: RETRY_DELAY - value: "0" - - name: TASK_HOST - value: "redis://localhost:6379/0" - - name: BANZAI_WORKER_LOGLEVEL - value: debug - - name: CALIBRATE_PROPOSAL_ID - value: "calibrate" - - name: OBSERVATION_PORTAL_URL - value: "http://internal-observation-portal.lco.gtn/api/observations/" - - name: API_ROOT - value: "https://archive-api.lco.global/" - - name: OMP_NUM_THREADS - value: "2" - - name: FITS_EXCHANGE - value: "fits_files" - - name: OPENTSDB_PYTHON_METRICS_TEST_MODE - value: "1" - - name: CELERY_TASK_QUEUE_NAME - value: "e2e_large_task_queue" - - name: REFERENCE_CATALOG_URL - value: "http://phot-catalog.lco.gtn/" - command: - - celery - - -A - - banzai - - worker - - --hostname - - "banzai-celery-worker" - - --concurrency - - "1" - - -l - - "info" - - "-Q" - - "$(CELERY_TASK_QUEUE_NAME)" - - "-n" - - "large-celery-worker" - readinessProbe: - exec: - command: - - celery - - -A - - banzai - - status - - | - - grep - - "banzai-celery-worker:.*OK" - initialDelaySeconds: 5 - periodSeconds: 1 - timeoutSeconds: 10 - resources: - requests: - cpu: 2 - memory: 8Gi - limits: - cpu: 3 - memory: 9Gi - - name: banzai-listener - image: banzai:test-latest - imagePullPolicy: IfNotPresent - volumeMounts: - - name: banzai-data - mountPath: /archive/engineering - subPath: engineering - readOnly: false - env: - - name: DB_ADDRESS - value: "sqlite:////archive/engineering/test.db" - - name: FITS_BROKER - value: "localhost" - - name: TASK_HOST - value: "redis://localhost:6379/0" - - name: CALIBRATE_PROPOSAL_ID - value: "calibrate" - - name: OBSERVATION_PORTAL_URL - value: "http://internal-observation-portal.lco.gtn/api/observations/" - - name: API_ROOT - value: "https://archive-api.lco.global/" - - name: FITS_EXCHANGE - value: "fits_files" - - name: OPENTSDB_PYTHON_METRICS_TEST_MODE - value: "1" - - name: CELERY_TASK_QUEUE_NAME - value: "e2e_task_queue" - - name: REFERENCE_CATALOG_URL - value: "http://phot-catalog.lco.gtn/" - - name: "CELERY_LARGE_TASK_QUEUE_NAME" - value: "e2e_large_task_queue" - command: - - banzai_run_realtime_pipeline - - "--db-address=sqlite:////archive/engineering/test.db" - - "--fpack" - - "--broker-url=localhost" - resources: - requests: - cpu: 0.1 - memory: 1Gi - limits: - cpu: 1 - memory: 2Gi + - name: banzai-redis + image: redis:5.0.3 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1 + memory: 1Gi + readinessProbe: + exec: + command: + - redis-cli + - ping + - | + - grep + - "PONG" + initialDelaySeconds: 5 + periodSeconds: 1 + - name: banzai-fits-exchange + image: rabbitmq:3.12.3 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 2 + memory: 1Gi + readinessProbe: + exec: + command: + - sh + - -c + - | + rabbitmq-diagnostics -q ping | grep "Ping succeeded" + initialDelaySeconds: 5 + periodSeconds: 1 + timeoutSeconds: 10 + - name: banzai-celery-workers + image: 127.0.0.1:5001/banzai + imagePullPolicy: IfNotPresent + volumeMounts: + - name: banzai-data + mountPath: /archive/engineering + subPath: engineering + readOnly: false + env: + - name: DB_ADDRESS + value: "sqlite:////archive/engineering/test.db" + - name: RETRY_DELAY + value: "0" + - name: TASK_HOST + value: "redis://localhost:6379/0" + - name: BANZAI_WORKER_LOGLEVEL + value: debug + - name: CALIBRATE_PROPOSAL_ID + value: "calibrate" + - name: OBSERVATION_PORTAL_URL + value: "http://internal-observation-portal.lco.gtn/api/observations/" + - name: OMP_NUM_THREADS + value: "2" + - name: FITS_EXCHANGE + value: "fits_files" + - name: OPENTSDB_PYTHON_METRICS_TEST_MODE + value: "1" + - name: CELERY_TASK_QUEUE_NAME + value: "e2e_task_queue" + - name: REFERENCE_CATALOG_URL + value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- + command: + - celery + - -A + - banzai + - worker + - --hostname + - "banzai-celery-worker" + - --concurrency + - "2" + - -l + - "info" + - "-Q" + - "$(CELERY_TASK_QUEUE_NAME)" + - "-n" + - "celery-worker" + readinessProbe: + exec: + command: + - sh + - -c + - | + celery -A banzai status | grep "banzai-celery-worker:.*OK" + initialDelaySeconds: 5 + periodSeconds: 1 + timeoutSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 8 + memory: 8Gi + - name: banzai-large-celery-workers + image: 127.0.0.1:5001/banzai + imagePullPolicy: IfNotPresent + volumeMounts: + - name: banzai-data + mountPath: /archive/engineering + subPath: engineering + readOnly: false + env: + - name: DB_ADDRESS + value: "sqlite:////archive/engineering/test.db" + - name: RETRY_DELAY + value: "0" + - name: TASK_HOST + value: "redis://localhost:6379/0" + - name: BANZAI_WORKER_LOGLEVEL + value: debug + - name: CALIBRATE_PROPOSAL_ID + value: "calibrate" + - name: OBSERVATION_PORTAL_URL + value: "http://internal-observation-portal.lco.gtn/api/observations/" + - name: OMP_NUM_THREADS + value: "2" + - name: FITS_EXCHANGE + value: "fits_files" + - name: OPENTSDB_PYTHON_METRICS_TEST_MODE + value: "1" + - name: CELERY_TASK_QUEUE_NAME + value: "e2e_large_task_queue" + - name: REFERENCE_CATALOG_URL + value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- + command: + - celery + - -A + - banzai + - worker + - --hostname + - "banzai-celery-worker" + - --concurrency + - "1" + - -l + - "info" + - "-Q" + - "$(CELERY_TASK_QUEUE_NAME)" + - "-n" + - "large-celery-worker" + - name: banzai-celery-beat + image: 127.0.0.1:5001/banzai + imagePullPolicy: IfNotPresent + volumeMounts: + - name: banzai-data + mountPath: /archive/engineering + subPath: engineering + readOnly: false + env: + - name: DB_ADDRESS + value: "sqlite:////archive/engineering/test.db" + - name: RETRY_DELAY + value: "0" + - name: TASK_HOST + value: "redis://localhost:6379/0" + - name: CELERY_TASK_QUEUE_NAME + value: "e2e_task_queue" + command: + - banzai_automate_stack_calibrations + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1 + memory: 1Gi + - name: banzai-listener + image: 127.0.0.1:5001/banzai + imagePullPolicy: IfNotPresent + volumeMounts: + - name: banzai-data + mountPath: /archive/engineering + subPath: engineering + readOnly: false + env: + - name: DB_ADDRESS + value: "sqlite:////archive/engineering/test.db" + - name: FITS_BROKER + value: "localhost" + - name: TASK_HOST + value: "redis://localhost:6379/0" + - name: CALIBRATE_PROPOSAL_ID + value: "calibrate" + - name: OBSERVATION_PORTAL_URL + value: "http://internal-observation-portal.lco.gtn/api/observations/" + - name: FITS_EXCHANGE + value: "fits_files" + - name: OPENTSDB_PYTHON_METRICS_TEST_MODE + value: "1" + - name: CELERY_TASK_QUEUE_NAME + value: "e2e_task_queue" + - name: "CELERY_LARGE_TASK_QUEUE_NAME" + value: "e2e_large_task_queue" + - name: REFERENCE_CATALOG_URL + value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- + command: + - banzai_run_realtime_pipeline + - "--db-address=sqlite:////archive/engineering/test.db" + - "--fpack" + - "--broker-url=localhost" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1 + memory: 2Gi dnsPolicy: ClusterFirst restartPolicy: Never diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..a6c1801e --- /dev/null +++ b/flake.lock @@ -0,0 +1,761 @@ +{ + "nodes": { + "cachix": { + "inputs": { + "devenv": "devenv_2", + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1712055811, + "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", + "owner": "cachix", + "repo": "cachix", + "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + "deploy-repo-template": { + "flake": false, + "locked": { + "lastModified": 1710788754, + "narHash": "sha256-46GUCjs0oPZp2AxzHq6OZITzmqwwhb3Pbgtno6Ec9zY=", + "ref": "refs/heads/main", + "rev": "79e85fc90d7d1d0c3c5e4b1db676b2de377a69e7", + "revCount": 14, + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + }, + "original": { + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + } + }, + "devenv": { + "inputs": { + "cachix": "cachix", + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": "nixpkgs_2", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1713195025, + "narHash": "sha256-TIUCx0Vlu9ImxTtRyQUFf5RWvTT4GxCTKRSiMFZdZuw=", + "owner": "cachix", + "repo": "devenv", + "rev": "01e01a3b4d991983dda055d10dcf93e86e1e1f8a", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv-k8s": { + "inputs": { + "deploy-repo-template": "deploy-repo-template", + "devenv": "devenv", + "flake-parts": "flake-parts", + "kpt": "kpt", + "mk-shell-bin": "mk-shell-bin", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_3", + "octopilot": "octopilot", + "skaffold": "skaffold" + }, + "locked": { + "lastModified": 1713222238, + "narHash": "sha256-viSH6JTcPJxu/BQnXBjgUsmhJatCxlpmqMizrI2DuYk=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "6c9d9b168c07ac6d7592c2543534c70f5586b453", + "type": "github" + }, + "original": { + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "cachix", + "flake-compat" + ], + "nix": "nix", + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix", + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "cachix", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1708704632, + "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", + "owner": "cachix", + "repo": "devenv", + "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "python-rewrite", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": "flake-utils_4", + "nixpkgs": [ + "devenv-k8s", + "skaffold", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1710154385, + "narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "872b63ddd28f318489c929d25f1f0a3c6039c971", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "gomod2nix", + "type": "github" + } + }, + "kpt": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "kpt": "kpt_2", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "dir": "kpt", + "lastModified": 1712953401, + "narHash": "sha256-A/b71BZrx+FbGLtO/hAcvOeBI8r6hM0XDccoa3IgaCE=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "d9556f6cd69348b6f726c217a1f53638f6594917", + "type": "github" + }, + "original": { + "dir": "kpt", + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" + } + }, + "kpt_2": { + "flake": false, + "locked": { + "lastModified": 1700611672, + "narHash": "sha256-GHEk5nezva+26cpCrgriaGIL2PvSLWgC0UtmFkNHoDc=", + "owner": "kptdev", + "repo": "kpt", + "rev": "aff697dbfc8134b059bcfbdfb792a1048aaa57b5", + "type": "github" + }, + "original": { + "owner": "kptdev", + "ref": "v1.0.0-beta.48", + "repo": "kpt", + "type": "github" + } + }, + "mk-shell-bin": { + "locked": { + "lastModified": 1677004959, + "narHash": "sha256-/uEkr1UkJrh11vD02aqufCxtbF5YnhRTIKlx5kyvf+I=", + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", + "type": "github" + }, + "original": { + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "type": "github" + } + }, + "nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1708577783, + "narHash": "sha256-92xq7eXlxIT5zFNccLpjiP7sdQqQI30Gyui2p/PfKZM=", + "owner": "domenkozar", + "repo": "nix", + "rev": "ecd0af0c1f56de32cbad14daa1d82a132bf298f8", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nix_2": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1692808169, + "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1710695816, + "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "614b4613980a522ba49f0d194531beddbb7220d3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1710796454, + "narHash": "sha256-lQlICw60RhH8sHTDD/tJiiJrlAfNn8FDI9c+7G2F0SE=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "06fb0f1c643aee3ae6838dda3b37ef0abc3c763b", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1712963716, + "narHash": "sha256-WKm9CvgCldeIVvRz87iOMi8CFVB1apJlkUT4GGvA0iM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "cfd6b5fc90b15709b780a5a1619695a88505a176", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "octopilot": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1711718101, + "narHash": "sha256-bMeByl+3OdGsFcHA2fE37IvSbcG81SYhuruZzJ/YPdA=", + "owner": "dailymotion-oss", + "repo": "octopilot", + "rev": "319bb180ec99b5a8ac5d23d75dedfeb61d5feeaf", + "type": "github" + }, + "original": { + "owner": "dailymotion-oss", + "repo": "octopilot", + "type": "github" + } + }, + "poetry2nix": { + "inputs": { + "flake-utils": "flake-utils", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1692876271, + "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils_2", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1712897695, + "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv-k8s": "devenv-k8s", + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + } + }, + "skaffold": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "gomod2nix": "gomod2nix", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1711736775, + "narHash": "sha256-wRzy040KH+IfgK+g3Dj4Re4hb1S+G3xaxJ41TrsPEvE=", + "owner": "jashandeep-sohi", + "repo": "skaffold", + "rev": "e57813c8e738a0a347f8732221961584e025b6a5", + "type": "github" + }, + "original": { + "owner": "jashandeep-sohi", + "repo": "skaffold", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_4": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..2a12ec1c --- /dev/null +++ b/flake.nix @@ -0,0 +1,110 @@ +{ + description = "Description for the project"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + devenv-k8s.url = "github:LCOGT/devenv-k8s"; + + nixpkgs.follows = "devenv-k8s/nixpkgs"; + flake-parts.follows = "devenv-k8s/flake-parts"; + }; + + nixConfig = { + extra-substituters = [ + "https://devenv.cachix.org" + "https://lco-public.cachix.org" + ]; + + extra-trusted-public-keys = [ + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + "lco-public.cachix.org-1:zSmLK7CkAehZ7QzTLZKt+5Y26Lr0w885GUB4GlT1SCg=" + ]; + }; + + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + inputs.devenv-k8s.flakeModules.default + ]; + + systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ]; + + perSystem = { config, self', inputs', pkgs, system, ... }: { + # Per-system attributes can be defined here. The self' and inputs' + # module parameters provide easy access to attributes of the same + # system. + + # https://devenv.sh/basics/ + # Enter using `nix develop --impure` + config.devenv.shells.default = { + + # https://devenv.sh/packages/ + packages = [ + + ]; + + scripts.kind-create-cluster.exec = '' + set -o errexit + + # # Create the Kind cluster + kind create cluster --config ./kind-cluster.yaml + + # Create registry container unless it already exists + reg_name='kind-registry' + reg_port='5001' + + if [ "$(docker inspect -f '{{.State.Running}}' "''${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:''${reg_port}:5000" --network bridge --name "''${reg_name}" \ + registry:2 + fi + + # 3. Add the registry config to the nodes + # + # This is necessary because localhost resolves to loopback addresses that are + # network-namespace local. + # In other words: localhost in the container is not localhost on the host. + # + # We want a consistent name that works from both ends, so we tell containerd to + # alias localhost:reg_port to the registry container when pulling images + REGISTRY_DIR="/etc/containerd/certs.d/127.0.0.1:''${reg_port}" + for node in $(kind get nodes); do + docker exec "''${node}" mkdir -p "''${REGISTRY_DIR}" + cat </dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:''${reg_port}:5000" --network bridge --name "''${reg_name}" \ + registry:2 + fi + + # 3. Add the registry config to the nodes + # + # This is necessary because localhost resolves to loopback addresses that are + # network-namespace local. + # In other words: localhost in the container is not localhost on the host. + # + # We want a consistent name that works from both ends, so we tell containerd to + # alias localhost:reg_port to the registry container when pulling images + REGISTRY_DIR="/etc/containerd/certs.d/localhost:''${reg_port}" + for node in $(kind get nodes); do + docker exec "''${node}" mkdir -p "''${REGISTRY_DIR}" + cat <=4.0 pyyaml psycopg2-binary - celery[redis]>=4.3.1,<5 + celery[redis]>5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000..39479de2 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,24 @@ +apiVersion: skaffold/v4beta8 +kind: Config +metadata: + name: banzai +profiles: + - name: e2e + build: + artifacts: + - image: 127.0.0.1:5001/banzai + context: . + custom: + buildCommand: skaffold-builder-buildx + dependencies: + dockerfile: + path: Dockerfile + local: + push: true + + manifests: + rawYaml: + - ./banzai/tests/e2e-k8s.yaml + + deploy: + kubectl: {}