From 24e87571ccb2543cd2c808b34646036d1f65397d Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 5 Dec 2023 12:06:15 -0800 Subject: [PATCH 01/17] add some caching for docker builds --- .github/workflows/e2e.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1b96cb1d..edfb1087 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -9,6 +9,12 @@ jobs: runs-on: - banzai-runner steps: + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Checkout uses: actions/checkout@v4 @@ -27,7 +33,7 @@ jobs: - name: Build docker image run: | - docker build -t banzai:test-latest . + docker buildx build -t banzai:test-latest --cache-to type=gha,ignore-error=true,mode=max --cache-from type=gha . - name: Copy docker image to nodes run: | From 8e87d7b01af169e3d5fd4ffedce09fbeaa936063 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 5 Dec 2023 12:22:54 -0800 Subject: [PATCH 02/17] buildx must also export --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index edfb1087..e3ea19ea 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -33,7 +33,7 @@ jobs: - name: Build docker image run: | - docker buildx build -t banzai:test-latest --cache-to type=gha,ignore-error=true,mode=max --cache-from type=gha . + docker buildx build -t banzai:test-latest --load --cache-to type=gha,ignore-error=true,mode=max --cache-from type=gha . - name: Copy docker image to nodes run: | From c467dd89d4004024b8af48066cfee45749aff5a2 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 5 Dec 2023 12:54:48 -0800 Subject: [PATCH 03/17] buildx doesn't play nice with dind --- .github/workflows/e2e.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e3ea19ea..1b96cb1d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -9,12 +9,6 @@ jobs: runs-on: - banzai-runner steps: - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Checkout uses: actions/checkout@v4 @@ -33,7 +27,7 @@ jobs: - name: Build docker image run: | - docker buildx build -t banzai:test-latest --load --cache-to type=gha,ignore-error=true,mode=max --cache-from type=gha . + docker build -t banzai:test-latest . - name: Copy docker image to nodes run: | From 7956dec8e40e7e7baa209ead0b5401f2e2978fe7 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 5 Dec 2023 13:00:11 -0800 Subject: [PATCH 04/17] start kind after building image --- .github/workflows/e2e.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1b96cb1d..26f738de 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -17,6 +17,10 @@ jobs: with: version: "v1.22.17" + - name: Build docker image + run: | + docker build -t banzai:test-latest . + - name: Start a Kind cluster uses: helm/kind-action@v1.8.0 with: @@ -25,10 +29,6 @@ jobs: node_image: kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5 wait: "600s" - - name: Build docker image - run: | - docker build -t banzai:test-latest . - - name: Copy docker image to nodes run: | kind load docker-image banzai:test-latest From 3f8c2e9384ade85d796a8bb1e8697ee8f2f243f1 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Wed, 6 Dec 2023 11:05:33 -0800 Subject: [PATCH 05/17] use nix/flake to install all tools --- .dockerignore | 4 + .github/workflows/e2e.yaml | 47 ++- .gitignore | 2 + banzai/tests/e2e-k8s.yaml | 33 +- flake.lock | 845 +++++++++++++++++++++++++++++++++++++ flake.nix | 107 +++++ kind-cluster.yaml | 9 + skaffold.yaml | 24 ++ 8 files changed, 1028 insertions(+), 43 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 kind-cluster.yaml create mode 100644 skaffold.yaml diff --git a/.dockerignore b/.dockerignore index 85f331d9..a65e23a2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,3 +73,7 @@ distribute-*.tar.gz *.xml *.iml + +.venv/ + +.devenv/ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 26f738de..3ae2f932 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,37 +12,38 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Install kubectl - uses: azure/setup-kubectl@v3 + - name: Install nix + uses: cachix/install-nix-action@v24 with: - version: "v1.22.17" + extra_nix_config: | + accept-flake-config = true - - name: Build docker image - run: | - docker build -t banzai:test-latest . - - - name: Start a Kind cluster - uses: helm/kind-action@v1.8.0 + - name: Setup cachix + uses: cachix/cachix-action@v12 with: - cluster_name: kind - version: "v0.19.0" - node_image: kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5 - wait: "600s" + name: lco-public + extraPullNames: devenv + skipPush: true - - name: Copy docker image to nodes - run: | - kind load docker-image banzai:test-latest + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 - - name: Start banzai - run: | - cat banzai/tests/e2e-k8s.yaml + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - # Deploy banzai stack - kubectl apply -f banzai/tests/e2e-k8s.yaml + - name: Start Kind cluster + shell: nix develop --impure --command bash -e {0} + run: | + kind-create-cluster - # Wait for banzai to be ready - kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test + - name: Deploy Banzai w/ Skaffold + env: + SKAFFOLD_BUILDX_ARGS: "--cache-to type=gha,mode=max,ignore-error=true --cache-from type=gha" + shell: nix develop --impure --command bash -e {0} + run: | + skaffold -p e2e run - name: Test Master Bias Creation + shell: nix develop --impure --command bash -e {0} run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias diff --git a/.gitignore b/.gitignore index acb9875b..168e0832 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,5 @@ distribute-*.tar.gz *.iml .coverage.subprocess + +.devenv/ diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index fac1ccec..174c6125 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -31,13 +31,6 @@ spec: - -a - /archive/engineering - /data/ - resources: - requests: - cpu: 0.1 - memory: 256M - limits: - cpu: 1 - memory: 1Gi containers: - name: banzai-redis @@ -45,8 +38,8 @@ spec: imagePullPolicy: IfNotPresent resources: requests: - cpu: 0.1 - memory: 256M + cpu: 100m + memory: 256Mi limits: cpu: 1 memory: 1Gi @@ -65,8 +58,8 @@ spec: imagePullPolicy: IfNotPresent resources: requests: - cpu: 1 - memory: 512M + cpu: 100m + memory: 256Mi limits: cpu: 2 memory: 1Gi @@ -79,7 +72,7 @@ spec: periodSeconds: 1 timeoutSeconds: 10 - name: banzai-celery-workers - image: banzai:test-latest + image: localhost:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -137,13 +130,13 @@ spec: timeoutSeconds: 10 resources: requests: - cpu: 6 - memory: 6Gi + cpu: 100m + memory: 256Mi limits: cpu: 8 memory: 8Gi - name: banzai-celery-beat - image: banzai:test-latest + image: localhost:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -163,13 +156,13 @@ spec: - banzai_automate_stack_calibrations resources: requests: - cpu: 0.1 - memory: 1Gi + cpu: 100m + memory: 256Mi limits: cpu: 1 memory: 1Gi - name: banzai-listener - image: banzai:test-latest + image: localhost:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -202,8 +195,8 @@ spec: - "--broker-url=localhost" resources: requests: - cpu: 0.1 - memory: 1Gi + cpu: 100m + memory: 256Mi limits: cpu: 1 memory: 2Gi diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..71c2890d --- /dev/null +++ b/flake.lock @@ -0,0 +1,845 @@ +{ + "nodes": { + "devenv": { + "inputs": { + "flake-compat": "flake-compat", + "nix": "nix", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1701187605, + "narHash": "sha256-NctguPdUeDVLXFsv6vI1RlEiHLsXkeW3pgZe/mwn1BU=", + "owner": "cachix", + "repo": "devenv", + "rev": "a7c4dd8f4eb1f98a6b8f04bf08364954e1e73e4f", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv-k8s": { + "inputs": { + "devenv": "devenv_2", + "flake-parts": "flake-parts", + "kpt": "kpt", + "mk-shell-bin": "mk-shell-bin", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_3", + "octopilot": "octopilot" + }, + "locked": { + "lastModified": 1701839595, + "narHash": "sha256-vUqdc9EeiF1QM4ZzoVLwBHvHSpV1U1w3k3wI5gnVdgo=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "a52beff1149c9927f973edf0d1d7d0dbe3c7544a", + "type": "github" + }, + "original": { + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": "nixpkgs_2", + "pre-commit-hooks": "pre-commit-hooks_2" + }, + "locked": { + "lastModified": 1701187605, + "narHash": "sha256-NctguPdUeDVLXFsv6vI1RlEiHLsXkeW3pgZe/mwn1BU=", + "owner": "cachix", + "repo": "devenv", + "rev": "a7c4dd8f4eb1f98a6b8f04bf08364954e1e73e4f", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_2" + }, + "locked": { + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-parts_3": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_3" + }, + "locked": { + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-parts_4": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_4" + }, + "locked": { + "lastModified": 1701473968, + "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "type": "github" + }, + "original": { + "id": "flake-parts", + "type": "indirect" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_3" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": "systems_4" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "gitignore_2": { + "inputs": { + "nixpkgs": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "kpt": { + "inputs": { + "flake-parts": "flake-parts_2", + "kpt": "kpt_2", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1, + "narHash": "sha256-tl+05LnmHU/E2z6CLnQbqMZHv34Qk0GxleGlQkpxqrs=", + "path": "kpt", + "type": "path" + }, + "original": { + "path": "kpt", + "type": "path" + } + }, + "kpt_2": { + "flake": false, + "locked": { + "lastModified": 1700611672, + "narHash": "sha256-GHEk5nezva+26cpCrgriaGIL2PvSLWgC0UtmFkNHoDc=", + "owner": "kptdev", + "repo": "kpt", + "rev": "aff697dbfc8134b059bcfbdfb792a1048aaa57b5", + "type": "github" + }, + "original": { + "owner": "kptdev", + "ref": "v1.0.0-beta.48", + "repo": "kpt", + "type": "github" + } + }, + "lowdown-src": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "lowdown-src_2": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, + "mk-shell-bin": { + "locked": { + "lastModified": 1677004959, + "narHash": "sha256-/uEkr1UkJrh11vD02aqufCxtbF5YnhRTIKlx5kyvf+I=", + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", + "type": "github" + }, + "original": { + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "type": "github" + } + }, + "mk-shell-bin_2": { + "locked": { + "lastModified": 1677004959, + "narHash": "sha256-/uEkr1UkJrh11vD02aqufCxtbF5YnhRTIKlx5kyvf+I=", + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", + "type": "github" + }, + "original": { + "owner": "rrbutani", + "repo": "nix-mk-shell-bin", + "type": "github" + } + }, + "nix": { + "inputs": { + "lowdown-src": "lowdown-src", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_3", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1700389764, + "narHash": "sha256-hMsZ741ri9c4ZQpB6mgLY8KErk0yXVAOUjfNkP1nbbw=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "4400b77e14f3095ee3215a9a5e0f9143bc0e8f2d", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nix2container_2": { + "inputs": { + "flake-utils": "flake-utils_4", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1700389764, + "narHash": "sha256-hMsZ741ri9c4ZQpB6mgLY8KErk0yXVAOUjfNkP1nbbw=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "4400b77e14f3095ee3215a9a5e0f9143bc0e8f2d", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nix_2": { + "inputs": { + "lowdown-src": "lowdown-src_2", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1678875422, + "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib_2": { + "locked": { + "dir": "lib", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib_3": { + "locked": { + "dir": "lib", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib_4": { + "locked": { + "dir": "lib", + "lastModified": 1701253981, + "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable_2": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1678875422, + "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1701436327, + "narHash": "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "91050ea1e57e50388fa87a3302ba12d188ef723a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1701436327, + "narHash": "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "91050ea1e57e50388fa87a3302ba12d188ef723a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "octopilot": { + "inputs": { + "flake-parts": "flake-parts_3", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ], + "octopilot": "octopilot_2" + }, + "locked": { + "lastModified": 1, + "narHash": "sha256-RweJL6JX9OjVHZEt64hbqb0GcuGt/3LEqwDlvTP/PxM=", + "path": "octopilot", + "type": "path" + }, + "original": { + "path": "octopilot", + "type": "path" + } + }, + "octopilot_2": { + "flake": false, + "locked": { + "lastModified": 1701682992, + "narHash": "sha256-OEtN4lpeLmA7rw0A4YpVg7FTHZPelOks9gg7AcXB4YU=", + "owner": "dailymotion-oss", + "repo": "octopilot", + "rev": "472d535bae959cf48011ab4d1749f3cbb25c7b84", + "type": "github" + }, + "original": { + "owner": "dailymotion-oss", + "ref": "v1.6.0", + "repo": "octopilot", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils", + "gitignore": "gitignore", + "nixpkgs": [ + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1688056373, + "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "pre-commit-hooks_2": { + "inputs": { + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils_2", + "gitignore": "gitignore_2", + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable_2" + }, + "locked": { + "lastModified": 1688056373, + "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "devenv-k8s": "devenv-k8s", + "flake-parts": "flake-parts_4", + "mk-shell-bin": "mk-shell-bin_2", + "nix2container": "nix2container_2", + "nixpkgs": "nixpkgs_4" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_4": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..2f1f9b81 --- /dev/null +++ b/flake.nix @@ -0,0 +1,107 @@ +{ + description = "Banzai"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + devenv.url = "github:cachix/devenv"; + nix2container.url = "github:nlewo/nix2container"; + nix2container.inputs.nixpkgs.follows = "nixpkgs"; + mk-shell-bin.url = "github:rrbutani/nix-mk-shell-bin"; + devenv-k8s.url = "github:LCOGT/devenv-k8s"; + }; + + nixConfig = { + extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw="; + extra-substituters = "https://devenv.cachix.org"; + }; + + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + inputs.devenv.flakeModule + ]; + systems = [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; + + perSystem = { config, self', inputs', pkgs, system, ... }: { + # Per-system attributes can be defined here. The self' and inputs' + # module parameters provide easy access to attributes of the same + # system. + + devenv.shells.default = { + name = "banzai"; + + imports = [ + # This is just like the imports in devenv.nix. + # See https://devenv.sh/guides/using-with-flake-parts/#import-a-devenv-module + # ./devenv-foo.nix + ]; + + # https://devenv.sh/reference/options/ + packages = [ + inputs'.devenv-k8s.devShells.default + ]; + + scripts.kind-create-cluster.exec = '' + set -o errexit + + # Create the Kind cluster + kind create cluster --config ./kind-cluster.yaml + + # Create registry container unless it already exists + reg_name='kind-registry' + reg_port='5001' + + if [ "$(docker inspect -f '{{.State.Running}}' "''${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:''${reg_port}:5000" --network bridge --name "''${reg_name}" \ + registry:2 + fi + + # 3. Add the registry config to the nodes + # + # This is necessary because localhost resolves to loopback addresses that are + # network-namespace local. + # In other words: localhost in the container is not localhost on the host. + # + # We want a consistent name that works from both ends, so we tell containerd to + # alias localhost:reg_port to the registry container when pulling images + REGISTRY_DIR="/etc/containerd/certs.d/localhost:''${reg_port}" + for node in $(kind get nodes); do + docker exec "''${node}" mkdir -p "''${REGISTRY_DIR}" + cat < Date: Wed, 6 Dec 2023 11:17:51 -0800 Subject: [PATCH 06/17] debug --- .github/workflows/e2e.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3ae2f932..f61083ab 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -9,6 +9,9 @@ jobs: runs-on: - banzai-runner steps: + - name: Env + run: env + - name: Checkout uses: actions/checkout@v4 @@ -28,19 +31,22 @@ jobs: - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Start Kind cluster shell: nix develop --impure --command bash -e {0} run: | kind-create-cluster + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: "network=host" + - name: Deploy Banzai w/ Skaffold env: SKAFFOLD_BUILDX_ARGS: "--cache-to type=gha,mode=max,ignore-error=true --cache-from type=gha" shell: nix develop --impure --command bash -e {0} run: | + docker ps skaffold -p e2e run - name: Test Master Bias Creation From 3daf0f8b949140ff4cc424ecdcf01e5de4d74990 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Wed, 6 Dec 2023 15:42:40 -0800 Subject: [PATCH 07/17] add other tests --- .github/workflows/e2e.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f61083ab..9a7f898d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -9,9 +9,6 @@ jobs: runs-on: - banzai-runner steps: - - name: Env - run: env - - name: Checkout uses: actions/checkout@v4 @@ -53,3 +50,18 @@ jobs: shell: nix develop --impure --command bash -e {0} run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + + - name: Test Master Dark Creation + shell: nix develop --impure --command bash -e {0} + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark + + - name: Test Master Flat Creation + shell: nix develop --impure --command bash -e {0} + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + + - name: Test Science File Creation + shell: nix develop --impure --command bash -e {0} + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files From 4489497a7b51f7abd36ddd311481bdaa584e8bba Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 24 Jul 2024 10:27:10 -0400 Subject: [PATCH 08/17] Added support for stacked images. --- .github/workflows/e2e.yaml | 14 ++++++++++- banzai/bias.py | 6 ++--- banzai/data.py | 6 ++++- banzai/lco.py | 2 +- banzai/readnoise.py | 3 ++- banzai/settings.py | 4 +-- banzai/tests/data/test_data.dat | 43 +++++++++++++-------------------- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d43efe84..3fc9086c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -43,10 +43,22 @@ jobs: # Wait for banzai to be ready kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test - - name: Test Master Bias Creation + - name: Test Super Bias Creation run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + - name: Test Super Dark Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark + + - name: Test Super Flat Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat + + - name: Test Science Frame Creation + run: | + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files + - name: Cleanup run: | kubectl delete pod banzai-e2e-test diff --git a/banzai/bias.py b/banzai/bias.py index 19429727..316f166f 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -31,9 +31,9 @@ def calibration_type(self): return 'bias' def apply_master_calibration(self, image, master_calibration_image): - image -= master_calibration_image.bias_level + image -= master_calibration_image.bias_level * image.n_sub_exposures image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan' - image -= master_calibration_image + image -= master_calibration_image * image.n_sub_exposures image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame' image.meta['L1STATBI'] = 1, "Status flag for bias frame correction" return image @@ -65,7 +65,7 @@ def __init__(self, runtime_context): def do_stage(self, image): bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask) image -= bias_level - image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan' + image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan' return image diff --git a/banzai/data.py b/banzai/data.py index 4adf7322..2cda4f7d 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -132,7 +132,7 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header, mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True): super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap) if uncertainty is None: - uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain + uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain self.uncertainty = self._init_array(uncertainty) self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC')) self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC')) @@ -302,6 +302,10 @@ def data_section(self, section): self.meta['DATASEC'] = section.to_region_keyword() self._data_section = section + @property + def n_sub_exposures(self): + return self.meta.get('NSUBREAD', 1) + def rebin(self, binning): # TODO: Implement me return self diff --git a/banzai/lco.py b/banzai/lco.py index 6f801318..67447d57 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: # update datasec/trimsec for fs01 if hdu.header.get('INSTRUME') == 'fs01': self._update_fs01_sections(hdu) - if hdu.data.dtype == np.uint16: + if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32: hdu.data = hdu.data.astype(np.float64) # check if we need to propagate any header keywords from the primary header if primary_hdu is not None: diff --git a/banzai/readnoise.py b/banzai/readnoise.py index ba11ab69..08950775 100644 --- a/banzai/readnoise.py +++ b/banzai/readnoise.py @@ -1,5 +1,6 @@ from banzai.calibrations import CalibrationUser from banzai.logs import format_exception, get_logger +import numpy as np logger = get_logger() @@ -8,7 +9,7 @@ class ReadNoiseLoader(CalibrationUser): def apply_master_calibration(self, image, master_calibration_image): try: for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus): - image_extension.add_uncertainty(readnoise_extension.data) + image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures)) except: logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image) return None diff --git a/banzai/settings.py b/banzai/settings.py index fdfb98b5..9c511253 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -54,8 +54,8 @@ EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'], 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker', 'banzai.dark.DarkComparer'], - 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector', - 'banzai.flats.FlatComparer'], + 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', + 'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'], 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None} diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat index f7bc38f7..91a6b880 100644 --- a/banzai/tests/data/test_data.dat +++ b/banzai/tests/data/test_data.dat @@ -304,29 +304,20 @@ frameid filename filter site instrument 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04 -56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30 -56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30 -56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30 -56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30 -56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30 -56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30 -56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30 -56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30 -56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30 -56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30 -56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30 -56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30 -56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30 -56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30 -56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30 -56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30 -56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30 -56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30 -56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30 -56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30 -56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30 -56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30 -56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30 -56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30 -56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30 -56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30 +73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34 +73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34 +73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34 +73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34 +73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34 +73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34 +73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 +73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34 +73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34 +73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34 +73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34 +73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34 +73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34 +73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34 +73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34 +73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34 +73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34 From 00dd08a68be71f5aee09a6253baa4c892cf6b689 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Wed, 24 Jul 2024 11:05:17 -0400 Subject: [PATCH 09/17] Fixes to tests. --- banzai/data.py | 9 +++++++++ banzai/frames.py | 11 ++++++++++- banzai/tests/test_munge.py | 4 +++- banzai/tests/utils.py | 11 +++++++++-- setup.cfg | 2 +- 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/banzai/data.py b/banzai/data.py index 2cda4f7d..eb43c6c5 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -155,6 +155,15 @@ def __imul__(self, value): self.meta['MAXLIN'] *= value return self + def __mul__(self, value): + output = CCDData(self.data * value, meta=self.meta.copy(), + name=self.name, uncertainty=self.uncertainty * value, + mask=self.mask.copy(), memmap=self.memmap) + output.meta['SATURATE'] *= value + output.meta['GAIN'] /= value + output.meta['MAXLIN'] *= value + return output + def __itruediv__(self, value): if isinstance(value, CCDData): self.uncertainty = np.abs(self.data / value.data) * \ diff --git a/banzai/frames.py b/banzai/frames.py index 093a5734..ea4bbfbb 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -195,7 +195,10 @@ def binning(self): return [min(x_binnings), min(y_binnings)] def __sub__(self, other): - return self.primary_hdu - other.primary_hdu + if isinstance(other, ObservationFrame): + return self.primary_hdu - other.primary_hdu + else: + return self.primary_hdu - other def __isub__(self, other): if isinstance(other, ObservationFrame): @@ -204,6 +207,12 @@ def __isub__(self, other): self.primary_hdu.__isub__(other) return self + def __mul__(self, other): + if isinstance(other, ObservationFrame): + return self.primary_hdu.__mul__(other.primary_hdu) + else: + return self.primary_hdu.__mul__(other) + def __imul__(self, other): if isinstance(other, ObservationFrame): self.primary_hdu.__imul__(other.primary_hdu) diff --git a/banzai/tests/test_munge.py b/banzai/tests/test_munge.py index 5cc2a7b2..14fea328 100644 --- a/banzai/tests/test_munge.py +++ b/banzai/tests/test_munge.py @@ -55,7 +55,9 @@ def test_defaults_do_not_override_header(): def test_image_no_saturate_header_or_default(): with pytest.raises(MissingSaturate): - LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[FakeCCDData(meta={'INSTRUME': 'blah27'})])) + fake_data = FakeCCDData(meta={'INSTRUME': 'blah27'}) + fake_data.meta.pop('SATURATE') + LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[fake_data])) def test_saturate_no_default_but_header(): diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index a147ee97..afd3aa61 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -43,7 +43,13 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype) else: self.uncertainty = uncertainty - + self.memmap = True + if 'SATURATE' not in self.meta: + self.meta['SATURATE'] = 65535.0 + if 'GAIN' not in self.meta: + self.meta['GAIN'] = 1.0 + if 'MAXLIN' not in self.meta: + self.meta['MAXLIN'] = 65535.0 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -63,7 +69,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - + self.n_sub_exposures = 1 for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -159,6 +165,7 @@ def __init__(self, id=0, site='', camera='', enclosure='', telescope='', type='' self.nx = 4096 self.ny = 4096 + class FakeCalImage: def __init__(self): self.frameid = 1234 diff --git a/setup.cfg b/setup.cfg index 8e7c6b29..28c222b0 100755 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pytest>=4.0 pyyaml psycopg2-binary - celery[redis]>=4.3.1,<5 + celery[redis]>5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 From 29a2e22b5debec5cd0f965ff850d20b923626158 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Jul 2024 11:50:09 -0700 Subject: [PATCH 10/17] Fixes to celery beat API calls after v5 upgrade --- banzai/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/banzai/main.py b/banzai/main.py index e0fc4e66..9a45430d 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -22,9 +22,9 @@ from banzai.utils import date_utils, stage_utils, import_utils, image_utils, fits_utils, file_utils from banzai.celery import process_image, app, schedule_calibration_stacking from banzai.data import DataProduct +from celery.apps.beat import Beat from celery.schedules import crontab import celery -import celery.bin.beat import requests logger = logs.get_logger() @@ -174,9 +174,9 @@ def start_stacking_scheduler(): schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)), queue=runtime_context.CELERY_TASK_QUEUE_NAME) - beat = celery.bin.beat.beat(app=app) + beat = Beat(app=app, loglevel='INFO', schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid') logger.info('Starting celery beat') - beat.run(schedule='/tmp/celerybeat-schedule', pidfile='/tmp/celerybeat.pid', working_directory='/tmp') + beat.run() def run_realtime_pipeline(): From c9a06ac191eefe79550d882568d35314b81b6353 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Jul 2024 12:20:25 -0700 Subject: [PATCH 11/17] Make sure we use nix for all commands. Fix merge issue. --- .github/workflows/e2e.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1a3aff79..382a3dcb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -47,26 +47,16 @@ jobs: skaffold -p e2e run - name: Test Super Bias Creation + shell: nix develop --impure --command bash -e {0} run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias + kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias - name: Test Super Dark Creation - run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark - - - name: Test Super Flat Creation - run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat - - - name: Test Science Frame Creation - run: | - kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files - - - name: Cleanup + shell: nix develop --impure --command bash -e {0} run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark - - name: Test Master Flat Creation + - name: Test Super Flat Creation shell: nix develop --impure --command bash -e {0} run: | kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai.tests --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat From 781deea2e2b5a4dbf4d6946b0428dc1607b82321 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Jul 2024 13:09:36 -0700 Subject: [PATCH 12/17] Try adding kubectl so we can debug the deployment --- .github/workflows/e2e.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 382a3dcb..b81a4533 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -28,6 +28,9 @@ jobs: - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - name: Install kubectl + uses: azure/setup-kubectl@v3 + - name: Start Kind cluster shell: nix develop --impure --command bash -e {0} run: | From 4131b9f8dfa39c700d921514842250259f0b05ca Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Jul 2024 13:16:18 -0700 Subject: [PATCH 13/17] Undo that, that was dumb as we install all deps via nix. --- .github/workflows/e2e.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b81a4533..382a3dcb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -28,9 +28,6 @@ jobs: - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 - - name: Install kubectl - uses: azure/setup-kubectl@v3 - - name: Start Kind cluster shell: nix develop --impure --command bash -e {0} run: | From 47ad38dbfeffc97809a17a95ff4b128e8f525485 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Jul 2024 17:39:37 -0700 Subject: [PATCH 14/17] WIP: Updating flake and updates for e2e tests for new celery change Main thing is using 127.0.0.1 instead of localhost for the local container registry plays nicely with MacOS' localhost resolution --- banzai/tests/e2e-k8s.yaml | 15 +- flake.lock | 570 ++++++++++++++++---------------------- flake.nix | 47 ++-- flake.nix.backup | 107 +++++++ skaffold.yaml | 2 +- 5 files changed, 382 insertions(+), 359 deletions(-) create mode 100644 flake.nix.backup diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 174c6125..16607fb4 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -72,7 +72,7 @@ spec: periodSeconds: 1 timeoutSeconds: 10 - name: banzai-celery-workers - image: localhost:5001/banzai + image: 127.0.0.1:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -118,13 +118,10 @@ spec: readinessProbe: exec: command: - - celery - - -A - - banzai - - status + - sh + - -c - | - - grep - - "banzai-celery-worker:.*OK" + celery status -d "banzai-celery-worker" | grep "banzai-celery-worker:.*OK" initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -136,7 +133,7 @@ spec: cpu: 8 memory: 8Gi - name: banzai-celery-beat - image: localhost:5001/banzai + image: 127.0.0.1:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data @@ -162,7 +159,7 @@ spec: cpu: 1 memory: 1Gi - name: banzai-listener - image: localhost:5001/banzai + image: 127.0.0.1:5001/banzai imagePullPolicy: IfNotPresent volumeMounts: - name: banzai-data diff --git a/flake.lock b/flake.lock index 71c2890d..a6c1801e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,18 +1,68 @@ { "nodes": { + "cachix": { + "inputs": { + "devenv": "devenv_2", + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "devenv-k8s", + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1712055811, + "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", + "owner": "cachix", + "repo": "cachix", + "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + "deploy-repo-template": { + "flake": false, + "locked": { + "lastModified": 1710788754, + "narHash": "sha256-46GUCjs0oPZp2AxzHq6OZITzmqwwhb3Pbgtno6Ec9zY=", + "ref": "refs/heads/main", + "rev": "79e85fc90d7d1d0c3c5e4b1db676b2de377a69e7", + "revCount": 14, + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + }, + "original": { + "type": "git", + "url": "https://github.com/LCOGT/deploy-repo-template.git" + } + }, "devenv": { "inputs": { - "flake-compat": "flake-compat", - "nix": "nix", - "nixpkgs": "nixpkgs", + "cachix": "cachix", + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": "nixpkgs_2", "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1701187605, - "narHash": "sha256-NctguPdUeDVLXFsv6vI1RlEiHLsXkeW3pgZe/mwn1BU=", + "lastModified": 1713195025, + "narHash": "sha256-TIUCx0Vlu9ImxTtRyQUFf5RWvTT4GxCTKRSiMFZdZuw=", "owner": "cachix", "repo": "devenv", - "rev": "a7c4dd8f4eb1f98a6b8f04bf08364954e1e73e4f", + "rev": "01e01a3b4d991983dda055d10dcf93e86e1e1f8a", "type": "github" }, "original": { @@ -23,20 +73,22 @@ }, "devenv-k8s": { "inputs": { - "devenv": "devenv_2", + "deploy-repo-template": "deploy-repo-template", + "devenv": "devenv", "flake-parts": "flake-parts", "kpt": "kpt", "mk-shell-bin": "mk-shell-bin", "nix2container": "nix2container", "nixpkgs": "nixpkgs_3", - "octopilot": "octopilot" + "octopilot": "octopilot", + "skaffold": "skaffold" }, "locked": { - "lastModified": 1701839595, - "narHash": "sha256-vUqdc9EeiF1QM4ZzoVLwBHvHSpV1U1w3k3wI5gnVdgo=", + "lastModified": 1713222238, + "narHash": "sha256-viSH6JTcPJxu/BQnXBjgUsmhJatCxlpmqMizrI2DuYk=", "owner": "LCOGT", "repo": "devenv-k8s", - "rev": "a52beff1149c9927f973edf0d1d7d0dbe3c7544a", + "rev": "6c9d9b168c07ac6d7592c2543534c70f5586b453", "type": "github" }, "original": { @@ -47,21 +99,33 @@ }, "devenv_2": { "inputs": { - "flake-compat": "flake-compat_2", - "nix": "nix_2", - "nixpkgs": "nixpkgs_2", - "pre-commit-hooks": "pre-commit-hooks_2" + "flake-compat": [ + "devenv-k8s", + "devenv", + "cachix", + "flake-compat" + ], + "nix": "nix", + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix", + "pre-commit-hooks": [ + "devenv-k8s", + "devenv", + "cachix", + "pre-commit-hooks" + ] }, "locked": { - "lastModified": 1701187605, - "narHash": "sha256-NctguPdUeDVLXFsv6vI1RlEiHLsXkeW3pgZe/mwn1BU=", + "lastModified": 1708704632, + "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", "owner": "cachix", "repo": "devenv", - "rev": "a7c4dd8f4eb1f98a6b8f04bf08364954e1e73e4f", + "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", "type": "github" }, "original": { "owner": "cachix", + "ref": "python-rewrite", "repo": "devenv", "type": "github" } @@ -85,11 +149,11 @@ "flake-compat_2": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -103,67 +167,17 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1701473968, - "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", "type": "github" }, "original": { - "id": "flake-parts", - "type": "indirect" - } - }, - "flake-parts_2": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib_2" - }, - "locked": { - "lastModified": 1701473968, - "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", "type": "github" - }, - "original": { - "id": "flake-parts", - "type": "indirect" - } - }, - "flake-parts_3": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib_3" - }, - "locked": { - "lastModified": 1701473968, - "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", - "type": "github" - }, - "original": { - "id": "flake-parts", - "type": "indirect" - } - }, - "flake-parts_4": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib_4" - }, - "locked": { - "lastModified": 1701473968, - "narHash": "sha256-YcVE5emp1qQ8ieHUnxt1wCZCC3ZfAS+SRRWZ2TMda7E=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "34fed993f1674c8d06d58b37ce1e0fe5eebcb9f5", - "type": "github" - }, - "original": { - "id": "flake-parts", - "type": "indirect" } }, "flake-utils": { @@ -171,11 +185,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", "owner": "numtide", "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", "type": "github" }, "original": { @@ -189,11 +203,11 @@ "systems": "systems_2" }, "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -207,11 +221,11 @@ "systems": "systems_3" }, "locked": { - "lastModified": 1694529238, - "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -241,17 +255,18 @@ "gitignore": { "inputs": { "nixpkgs": [ + "devenv-k8s", "devenv", "pre-commit-hooks", "nixpkgs" ] }, "locked": { - "lastModified": 1660459072, - "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", "owner": "hercules-ci", "repo": "gitignore.nix", - "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", "type": "github" }, "original": { @@ -260,32 +275,35 @@ "type": "github" } }, - "gitignore_2": { + "gomod2nix": { "inputs": { + "flake-utils": "flake-utils_4", "nixpkgs": [ "devenv-k8s", - "devenv", - "pre-commit-hooks", + "skaffold", "nixpkgs" ] }, "locked": { - "lastModified": 1660459072, - "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "lastModified": 1710154385, + "narHash": "sha256-4c3zQ2YY4BZOufaBJB4v9VBBeN2dH7iVdoJw8SDNCfI=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "872b63ddd28f318489c929d25f1f0a3c6039c971", "type": "github" }, "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", + "owner": "nix-community", + "repo": "gomod2nix", "type": "github" } }, "kpt": { "inputs": { - "flake-parts": "flake-parts_2", + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], "kpt": "kpt_2", "nixpkgs": [ "devenv-k8s", @@ -293,14 +311,19 @@ ] }, "locked": { - "lastModified": 1, - "narHash": "sha256-tl+05LnmHU/E2z6CLnQbqMZHv34Qk0GxleGlQkpxqrs=", - "path": "kpt", - "type": "path" + "dir": "kpt", + "lastModified": 1712953401, + "narHash": "sha256-A/b71BZrx+FbGLtO/hAcvOeBI8r6hM0XDccoa3IgaCE=", + "owner": "LCOGT", + "repo": "devenv-k8s", + "rev": "d9556f6cd69348b6f726c217a1f53638f6594917", + "type": "github" }, "original": { - "path": "kpt", - "type": "path" + "dir": "kpt", + "owner": "LCOGT", + "repo": "devenv-k8s", + "type": "github" } }, "kpt_2": { @@ -320,38 +343,6 @@ "type": "github" } }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, - "lowdown-src_2": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "mk-shell-bin": { "locked": { "lastModified": 1677004959, @@ -367,80 +358,72 @@ "type": "github" } }, - "mk-shell-bin_2": { - "locked": { - "lastModified": 1677004959, - "narHash": "sha256-/uEkr1UkJrh11vD02aqufCxtbF5YnhRTIKlx5kyvf+I=", - "owner": "rrbutani", - "repo": "nix-mk-shell-bin", - "rev": "ff5d8bd4d68a347be5042e2f16caee391cd75887", - "type": "github" - }, - "original": { - "owner": "rrbutani", - "repo": "nix-mk-shell-bin", - "type": "github" - } - }, "nix": { "inputs": { - "lowdown-src": "lowdown-src", + "flake-compat": "flake-compat", "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", "devenv", "nixpkgs" ], "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1676545802, - "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "lastModified": 1708577783, + "narHash": "sha256-92xq7eXlxIT5zFNccLpjiP7sdQqQI30Gyui2p/PfKZM=", "owner": "domenkozar", "repo": "nix", - "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "rev": "ecd0af0c1f56de32cbad14daa1d82a132bf298f8", "type": "github" }, "original": { "owner": "domenkozar", - "ref": "relaxed-flakes", + "ref": "devenv-2.21", "repo": "nix", "type": "github" } }, - "nix2container": { + "nix-github-actions": { "inputs": { - "flake-utils": "flake-utils_3", "nixpkgs": [ "devenv-k8s", + "devenv", + "cachix", + "devenv", + "poetry2nix", "nixpkgs" ] }, "locked": { - "lastModified": 1700389764, - "narHash": "sha256-hMsZ741ri9c4ZQpB6mgLY8KErk0yXVAOUjfNkP1nbbw=", - "owner": "nlewo", - "repo": "nix2container", - "rev": "4400b77e14f3095ee3215a9a5e0f9143bc0e8f2d", + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", "type": "github" }, "original": { - "owner": "nlewo", - "repo": "nix2container", + "owner": "nix-community", + "repo": "nix-github-actions", "type": "github" } }, - "nix2container_2": { + "nix2container": { "inputs": { - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_3", "nixpkgs": [ + "devenv-k8s", "nixpkgs" ] }, "locked": { - "lastModified": 1700389764, - "narHash": "sha256-hMsZ741ri9c4ZQpB6mgLY8KErk0yXVAOUjfNkP1nbbw=", + "lastModified": 1712990762, + "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", "owner": "nlewo", "repo": "nix2container", - "rev": "4400b77e14f3095ee3215a9a5e0f9143bc0e8f2d", + "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", "type": "github" }, "original": { @@ -451,7 +434,11 @@ }, "nix_2": { "inputs": { - "lowdown-src": "lowdown-src_2", + "flake-compat": [ + "devenv-k8s", + "devenv", + "flake-compat" + ], "nixpkgs": [ "devenv-k8s", "devenv", @@ -460,27 +447,27 @@ "nixpkgs-regression": "nixpkgs-regression_2" }, "locked": { - "lastModified": 1676545802, - "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", "owner": "domenkozar", "repo": "nix", - "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", "type": "github" }, "original": { "owner": "domenkozar", - "ref": "relaxed-flakes", + "ref": "devenv-2.21", "repo": "nix", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1678875422, - "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "lastModified": 1692808169, + "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", "type": "github" }, "original": { @@ -493,65 +480,11 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1701253981, - "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", - "type": "github" - }, - "original": { - "dir": "lib", - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib_2": { - "locked": { - "dir": "lib", - "lastModified": 1701253981, - "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", - "type": "github" - }, - "original": { - "dir": "lib", - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib_3": { - "locked": { - "dir": "lib", - "lastModified": 1701253981, - "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", - "type": "github" - }, - "original": { - "dir": "lib", - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib_4": { - "locked": { - "dir": "lib", - "lastModified": 1701253981, - "narHash": "sha256-ztaDIyZ7HrTAfEEUt9AtTDNoCYxUdSd6NrRHaYOIxtk=", + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e92039b55bcd58469325ded85d4f58dd5a4eaf58", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", "type": "github" }, "original": { @@ -596,75 +529,43 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1685801374, - "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "lastModified": 1710695816, + "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "rev": "614b4613980a522ba49f0d194531beddbb7220d3", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-stable_2": { - "locked": { - "lastModified": 1685801374, - "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.05", + "ref": "nixos-23.11", "repo": "nixpkgs", "type": "github" } }, "nixpkgs_2": { "locked": { - "lastModified": 1678875422, - "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "lastModified": 1710796454, + "narHash": "sha256-lQlICw60RhH8sHTDD/tJiiJrlAfNn8FDI9c+7G2F0SE=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "06fb0f1c643aee3ae6838dda3b37ef0abc3c763b", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", "type": "github" } }, "nixpkgs_3": { "locked": { - "lastModified": 1701436327, - "narHash": "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "91050ea1e57e50388fa87a3302ba12d188ef723a", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_4": { - "locked": { - "lastModified": 1701436327, - "narHash": "sha256-tRHbnoNI8SIM5O5xuxOmtSLnswEByzmnQcGGyNRjxsE=", + "lastModified": 1712963716, + "narHash": "sha256-WKm9CvgCldeIVvRz87iOMi8CFVB1apJlkUT4GGvA0iM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "91050ea1e57e50388fa87a3302ba12d188ef723a", + "rev": "cfd6b5fc90b15709b780a5a1619695a88505a176", "type": "github" }, "original": { @@ -676,70 +577,56 @@ }, "octopilot": { "inputs": { - "flake-parts": "flake-parts_3", + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], "nixpkgs": [ "devenv-k8s", "nixpkgs" - ], - "octopilot": "octopilot_2" + ] }, "locked": { - "lastModified": 1, - "narHash": "sha256-RweJL6JX9OjVHZEt64hbqb0GcuGt/3LEqwDlvTP/PxM=", - "path": "octopilot", - "type": "path" - }, - "original": { - "path": "octopilot", - "type": "path" - } - }, - "octopilot_2": { - "flake": false, - "locked": { - "lastModified": 1701682992, - "narHash": "sha256-OEtN4lpeLmA7rw0A4YpVg7FTHZPelOks9gg7AcXB4YU=", + "lastModified": 1711718101, + "narHash": "sha256-bMeByl+3OdGsFcHA2fE37IvSbcG81SYhuruZzJ/YPdA=", "owner": "dailymotion-oss", "repo": "octopilot", - "rev": "472d535bae959cf48011ab4d1749f3cbb25c7b84", + "rev": "319bb180ec99b5a8ac5d23d75dedfeb61d5feeaf", "type": "github" }, "original": { "owner": "dailymotion-oss", - "ref": "v1.6.0", "repo": "octopilot", "type": "github" } }, - "pre-commit-hooks": { + "poetry2nix": { "inputs": { - "flake-compat": [ - "devenv", - "flake-compat" - ], "flake-utils": "flake-utils", - "gitignore": "gitignore", + "nix-github-actions": "nix-github-actions", "nixpkgs": [ + "devenv-k8s", + "devenv", + "cachix", "devenv", "nixpkgs" - ], - "nixpkgs-stable": "nixpkgs-stable" + ] }, "locked": { - "lastModified": 1688056373, - "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "lastModified": 1692876271, + "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", "type": "github" }, "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", + "owner": "nix-community", + "repo": "poetry2nix", "type": "github" } }, - "pre-commit-hooks_2": { + "pre-commit-hooks": { "inputs": { "flake-compat": [ "devenv-k8s", @@ -747,20 +634,20 @@ "flake-compat" ], "flake-utils": "flake-utils_2", - "gitignore": "gitignore_2", + "gitignore": "gitignore", "nixpkgs": [ "devenv-k8s", "devenv", "nixpkgs" ], - "nixpkgs-stable": "nixpkgs-stable_2" + "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1688056373, - "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", + "lastModified": 1712897695, + "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", "type": "github" }, "original": { @@ -771,12 +658,41 @@ }, "root": { "inputs": { - "devenv": "devenv", "devenv-k8s": "devenv-k8s", - "flake-parts": "flake-parts_4", - "mk-shell-bin": "mk-shell-bin_2", - "nix2container": "nix2container_2", - "nixpkgs": "nixpkgs_4" + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + } + }, + "skaffold": { + "inputs": { + "flake-parts": [ + "devenv-k8s", + "flake-parts" + ], + "gomod2nix": "gomod2nix", + "nixpkgs": [ + "devenv-k8s", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1711736775, + "narHash": "sha256-wRzy040KH+IfgK+g3Dj4Re4hb1S+G3xaxJ41TrsPEvE=", + "owner": "jashandeep-sohi", + "repo": "skaffold", + "rev": "e57813c8e738a0a347f8732221961584e025b6a5", + "type": "github" + }, + "original": { + "owner": "jashandeep-sohi", + "repo": "skaffold", + "type": "github" } }, "systems": { diff --git a/flake.nix b/flake.nix index 2f1f9b81..2a12ec1c 100644 --- a/flake.nix +++ b/flake.nix @@ -1,50 +1,53 @@ { - description = "Banzai"; + description = "Description for the project"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - devenv.url = "github:cachix/devenv"; - nix2container.url = "github:nlewo/nix2container"; - nix2container.inputs.nixpkgs.follows = "nixpkgs"; - mk-shell-bin.url = "github:rrbutani/nix-mk-shell-bin"; + flake-parts.url = "github:hercules-ci/flake-parts"; devenv-k8s.url = "github:LCOGT/devenv-k8s"; + + nixpkgs.follows = "devenv-k8s/nixpkgs"; + flake-parts.follows = "devenv-k8s/flake-parts"; }; nixConfig = { - extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw="; - extra-substituters = "https://devenv.cachix.org"; + extra-substituters = [ + "https://devenv.cachix.org" + "https://lco-public.cachix.org" + ]; + + extra-trusted-public-keys = [ + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + "lco-public.cachix.org-1:zSmLK7CkAehZ7QzTLZKt+5Y26Lr0w885GUB4GlT1SCg=" + ]; }; outputs = inputs@{ flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } { imports = [ - inputs.devenv.flakeModule + inputs.devenv-k8s.flakeModules.default ]; - systems = [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]; + + systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ]; perSystem = { config, self', inputs', pkgs, system, ... }: { # Per-system attributes can be defined here. The self' and inputs' # module parameters provide easy access to attributes of the same # system. - devenv.shells.default = { - name = "banzai"; + # https://devenv.sh/basics/ + # Enter using `nix develop --impure` + config.devenv.shells.default = { - imports = [ - # This is just like the imports in devenv.nix. - # See https://devenv.sh/guides/using-with-flake-parts/#import-a-devenv-module - # ./devenv-foo.nix - ]; - - # https://devenv.sh/reference/options/ + # https://devenv.sh/packages/ packages = [ - inputs'.devenv-k8s.devShells.default + ]; scripts.kind-create-cluster.exec = '' set -o errexit - # Create the Kind cluster + # # Create the Kind cluster kind create cluster --config ./kind-cluster.yaml # Create registry container unless it already exists @@ -65,7 +68,7 @@ # # We want a consistent name that works from both ends, so we tell containerd to # alias localhost:reg_port to the registry container when pulling images - REGISTRY_DIR="/etc/containerd/certs.d/localhost:''${reg_port}" + REGISTRY_DIR="/etc/containerd/certs.d/127.0.0.1:''${reg_port}" for node in $(kind get nodes); do docker exec "''${node}" mkdir -p "''${REGISTRY_DIR}" cat </dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:''${reg_port}:5000" --network bridge --name "''${reg_name}" \ + registry:2 + fi + + # 3. Add the registry config to the nodes + # + # This is necessary because localhost resolves to loopback addresses that are + # network-namespace local. + # In other words: localhost in the container is not localhost on the host. + # + # We want a consistent name that works from both ends, so we tell containerd to + # alias localhost:reg_port to the registry container when pulling images + REGISTRY_DIR="/etc/containerd/certs.d/localhost:''${reg_port}" + for node in $(kind get nodes); do + docker exec "''${node}" mkdir -p "''${REGISTRY_DIR}" + cat < Date: Thu, 25 Jul 2024 10:11:50 -0700 Subject: [PATCH 15/17] Revert "Merge branch 'stacked-images' of github.com:LCOGT/banzai into actions-e2e" This reverts commit 321a547c3284a31b948e33f988188b5639084d30, reversing changes made to 433ecbfe7177630c109c17c12b6fc0bc2084c39f. --- banzai/bias.py | 6 ++--- banzai/data.py | 15 +----------- banzai/frames.py | 11 +-------- banzai/lco.py | 2 +- banzai/readnoise.py | 3 +-- banzai/settings.py | 4 +-- banzai/tests/data/test_data.dat | 43 ++++++++++++++++++++------------- banzai/tests/test_munge.py | 4 +-- banzai/tests/utils.py | 11 ++------- setup.cfg | 2 +- 10 files changed, 39 insertions(+), 62 deletions(-) diff --git a/banzai/bias.py b/banzai/bias.py index 316f166f..19429727 100755 --- a/banzai/bias.py +++ b/banzai/bias.py @@ -31,9 +31,9 @@ def calibration_type(self): return 'bias' def apply_master_calibration(self, image, master_calibration_image): - image -= master_calibration_image.bias_level * image.n_sub_exposures + image -= master_calibration_image.bias_level image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan' - image -= master_calibration_image * image.n_sub_exposures + image -= master_calibration_image image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame' image.meta['L1STATBI'] = 1, "Status flag for bias frame correction" return image @@ -65,7 +65,7 @@ def __init__(self, runtime_context): def do_stage(self, image): bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask) image -= bias_level - image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan' + image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan' return image diff --git a/banzai/data.py b/banzai/data.py index eb43c6c5..4adf7322 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -132,7 +132,7 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header, mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True): super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap) if uncertainty is None: - uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain + uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain self.uncertainty = self._init_array(uncertainty) self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC')) self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC')) @@ -155,15 +155,6 @@ def __imul__(self, value): self.meta['MAXLIN'] *= value return self - def __mul__(self, value): - output = CCDData(self.data * value, meta=self.meta.copy(), - name=self.name, uncertainty=self.uncertainty * value, - mask=self.mask.copy(), memmap=self.memmap) - output.meta['SATURATE'] *= value - output.meta['GAIN'] /= value - output.meta['MAXLIN'] *= value - return output - def __itruediv__(self, value): if isinstance(value, CCDData): self.uncertainty = np.abs(self.data / value.data) * \ @@ -311,10 +302,6 @@ def data_section(self, section): self.meta['DATASEC'] = section.to_region_keyword() self._data_section = section - @property - def n_sub_exposures(self): - return self.meta.get('NSUBREAD', 1) - def rebin(self, binning): # TODO: Implement me return self diff --git a/banzai/frames.py b/banzai/frames.py index ea4bbfbb..093a5734 100644 --- a/banzai/frames.py +++ b/banzai/frames.py @@ -195,10 +195,7 @@ def binning(self): return [min(x_binnings), min(y_binnings)] def __sub__(self, other): - if isinstance(other, ObservationFrame): - return self.primary_hdu - other.primary_hdu - else: - return self.primary_hdu - other + return self.primary_hdu - other.primary_hdu def __isub__(self, other): if isinstance(other, ObservationFrame): @@ -207,12 +204,6 @@ def __isub__(self, other): self.primary_hdu.__isub__(other) return self - def __mul__(self, other): - if isinstance(other, ObservationFrame): - return self.primary_hdu.__mul__(other.primary_hdu) - else: - return self.primary_hdu.__mul__(other) - def __imul__(self, other): if isinstance(other, ObservationFrame): self.primary_hdu.__imul__(other.primary_hdu) diff --git a/banzai/lco.py b/banzai/lco.py index 67447d57..6f801318 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: # update datasec/trimsec for fs01 if hdu.header.get('INSTRUME') == 'fs01': self._update_fs01_sections(hdu) - if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32: + if hdu.data.dtype == np.uint16: hdu.data = hdu.data.astype(np.float64) # check if we need to propagate any header keywords from the primary header if primary_hdu is not None: diff --git a/banzai/readnoise.py b/banzai/readnoise.py index 08950775..ba11ab69 100644 --- a/banzai/readnoise.py +++ b/banzai/readnoise.py @@ -1,6 +1,5 @@ from banzai.calibrations import CalibrationUser from banzai.logs import format_exception, get_logger -import numpy as np logger = get_logger() @@ -9,7 +8,7 @@ class ReadNoiseLoader(CalibrationUser): def apply_master_calibration(self, image, master_calibration_image): try: for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus): - image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures)) + image_extension.add_uncertainty(readnoise_extension.data) except: logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image) return None diff --git a/banzai/settings.py b/banzai/settings.py index 9c511253..fdfb98b5 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -54,8 +54,8 @@ EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'], 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker', 'banzai.dark.DarkComparer'], - 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', - 'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'], + 'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector', + 'banzai.flats.FlatComparer'], 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None} diff --git a/banzai/tests/data/test_data.dat b/banzai/tests/data/test_data.dat index 91a6b880..f7bc38f7 100644 --- a/banzai/tests/data/test_data.dat +++ b/banzai/tests/data/test_data.dat @@ -304,20 +304,29 @@ frameid filename filter site instrument 33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04 33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04 33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04 -73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34 -73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34 -73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34 -73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34 -73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34 -73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34 -73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34 -73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34 -73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34 -73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34 -73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34 -73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34 -73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34 -73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34 -73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34 -73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34 -73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34 +56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30 +56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30 +56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30 +56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30 +56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30 +56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30 +56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30 +56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30 +56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30 +56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30 +56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30 +56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30 +56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30 +56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30 +56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30 +56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30 +56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30 +56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30 +56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30 +56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30 +56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30 +56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30 +56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30 +56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30 +56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30 +56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30 diff --git a/banzai/tests/test_munge.py b/banzai/tests/test_munge.py index 14fea328..5cc2a7b2 100644 --- a/banzai/tests/test_munge.py +++ b/banzai/tests/test_munge.py @@ -55,9 +55,7 @@ def test_defaults_do_not_override_header(): def test_image_no_saturate_header_or_default(): with pytest.raises(MissingSaturate): - fake_data = FakeCCDData(meta={'INSTRUME': 'blah27'}) - fake_data.meta.pop('SATURATE') - LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[fake_data])) + LCOFrameFactory._init_saturate(FakeLCOObservationFrame(hdu_list=[FakeCCDData(meta={'INSTRUME': 'blah27'})])) def test_saturate_no_default_but_header(): diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index afd3aa61..a147ee97 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -43,13 +43,7 @@ def __init__(self, image_multiplier=1.0, nx=101, ny=103, name='test_image', read self.uncertainty = self.read_noise * np.ones(self.data.shape, dtype=self.data.dtype) else: self.uncertainty = uncertainty - self.memmap = True - if 'SATURATE' not in self.meta: - self.meta['SATURATE'] = 65535.0 - if 'GAIN' not in self.meta: - self.meta['GAIN'] = 1.0 - if 'MAXLIN' not in self.meta: - self.meta['MAXLIN'] = 65535.0 + for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -69,7 +63,7 @@ def __init__(self, hdu_list=None, file_path='/tmp/test_image.fits', instrument=N self._file_path = file_path self.is_bad = False self.hdu_order = ['SCI', 'CAT', 'BPM', 'ERR'] - self.n_sub_exposures = 1 + for keyword in kwargs: setattr(self, keyword, kwargs[keyword]) @@ -165,7 +159,6 @@ def __init__(self, id=0, site='', camera='', enclosure='', telescope='', type='' self.nx = 4096 self.ny = 4096 - class FakeCalImage: def __init__(self): self.frameid = 1234 diff --git a/setup.cfg b/setup.cfg index 28c222b0..8e7c6b29 100755 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pytest>=4.0 pyyaml psycopg2-binary - celery[redis]>5 + celery[redis]>=4.3.1,<5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 From 6a8aacf35b919ae54ce136df874ced82510221fc Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Thu, 25 Jul 2024 10:14:19 -0700 Subject: [PATCH 16/17] Upgrade celery version so that things build Old (<5) versions of celery have a project setup that's incompatible with newer versions of pip, so use this as an opportunity to upgrade celery. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8e7c6b29..28c222b0 100755 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ install_requires = pytest>=4.0 pyyaml psycopg2-binary - celery[redis]>=4.3.1,<5 + celery[redis]>5 apscheduler python-dateutil ocs_ingester>=3.0.4,<4.0.0 From d1d4d7d578f4ef777f4c0fcd3e12cf3de5392206 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Thu, 25 Jul 2024 12:50:05 -0700 Subject: [PATCH 17/17] Add API root and Auth token in auth token will need to be sed'd in to avoid committing it to version control --- banzai/tests/e2e-k8s.yaml | 89 +++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/banzai/tests/e2e-k8s.yaml b/banzai/tests/e2e-k8s.yaml index 16607fb4..b3fc11e9 100644 --- a/banzai/tests/e2e-k8s.yaml +++ b/banzai/tests/e2e-k8s.yaml @@ -13,25 +13,6 @@ spec: securityContext: fsGroup: 10000 - initContainers: - # When the Pod is initialized, copy all files within the container at path - # /archive/engineering into the empty data volume mounted at /data - - name: banzai-data - image: docker.lco.global/banzai-e2e-data:1.4.2 - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 10087 - runAsGroup: 10000 - volumeMounts: - - name: banzai-data - mountPath: /data - readOnly: false - command: - - /bin/cp - - -a - - /archive/engineering - - /data/ - containers: - name: banzai-redis image: redis:5.0.3 @@ -66,8 +47,10 @@ spec: readinessProbe: exec: command: - - rabbitmqctl - - node_health_check + - sh + - -c + - | + rabbitmq-diagnostics -q ping | grep "Ping succeeded" initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -102,6 +85,10 @@ spec: value: "e2e_task_queue" - name: REFERENCE_CATALOG_URL value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- command: - celery - -A @@ -115,13 +102,15 @@ spec: - "info" - "-Q" - "$(CELERY_TASK_QUEUE_NAME)" + - "-n" + - "celery-worker" readinessProbe: exec: command: - sh - -c - | - celery status -d "banzai-celery-worker" | grep "banzai-celery-worker:.*OK" + celery -A banzai status | grep "banzai-celery-worker:.*OK" initialDelaySeconds: 5 periodSeconds: 1 timeoutSeconds: 10 @@ -132,6 +121,56 @@ spec: limits: cpu: 8 memory: 8Gi + - name: banzai-large-celery-workers + image: 127.0.0.1:5001/banzai + imagePullPolicy: IfNotPresent + volumeMounts: + - name: banzai-data + mountPath: /archive/engineering + subPath: engineering + readOnly: false + env: + - name: DB_ADDRESS + value: "sqlite:////archive/engineering/test.db" + - name: RETRY_DELAY + value: "0" + - name: TASK_HOST + value: "redis://localhost:6379/0" + - name: BANZAI_WORKER_LOGLEVEL + value: debug + - name: CALIBRATE_PROPOSAL_ID + value: "calibrate" + - name: OBSERVATION_PORTAL_URL + value: "http://internal-observation-portal.lco.gtn/api/observations/" + - name: OMP_NUM_THREADS + value: "2" + - name: FITS_EXCHANGE + value: "fits_files" + - name: OPENTSDB_PYTHON_METRICS_TEST_MODE + value: "1" + - name: CELERY_TASK_QUEUE_NAME + value: "e2e_large_task_queue" + - name: REFERENCE_CATALOG_URL + value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- + command: + - celery + - -A + - banzai + - worker + - --hostname + - "banzai-celery-worker" + - --concurrency + - "1" + - -l + - "info" + - "-Q" + - "$(CELERY_TASK_QUEUE_NAME)" + - "-n" + - "large-celery-worker" - name: banzai-celery-beat image: 127.0.0.1:5001/banzai imagePullPolicy: IfNotPresent @@ -183,8 +222,14 @@ spec: value: "1" - name: CELERY_TASK_QUEUE_NAME value: "e2e_task_queue" + - name: "CELERY_LARGE_TASK_QUEUE_NAME" + value: "e2e_large_task_queue" - name: REFERENCE_CATALOG_URL value: "http://phot-catalog.lco.gtn/" + - name: API_ROOT + value: "http://archive-api.lco.global/" + - name: AUTH_TOKEN + value: -authtoken- command: - banzai_run_realtime_pipeline - "--db-address=sqlite:////archive/engineering/test.db"