Merge remote-tracking branch 'origin/master' into actions-e2e

.github/workflows/docker_image.yml

@@ -0,0 +1,53 @@
+name: Docker Image
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  build-tag:
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      packages: write
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Login to Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and also push Dockerimage
+        id: build-and-push
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/build.yml → .github/workflows/unit-tests.yml

@@ -12,39 +12,35 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Python 3.7 with required dependencies
-            os: macos-latest
-            python-version: 3.7
-            toxenv: py37-test
 
           - name: Documentation build
             os: ubuntu-latest
-            python-version: 3.8
+            python-version: 3.9
             toxenv: build_docs
 
           - name: Python 3.9 with developer version of astropy and numpy
             os: ubuntu-latest
             python-version: 3.9
             toxenv: py39-test-devdeps
 
-          - name: Python 3.7 with older dependencies, astropy 3.0 and Numpy 1.17
+          - name: Python 3.8 with minimal dependencies
             os: ubuntu-latest
-            python-version: 3.7
-            toxenv: py37-test-astropy30-numpy117
+            python-version: '3.8'
+            toxenv: py38-test
 
           - name: Python 3.9 with minimal dependencies
             os: ubuntu-latest
             python-version: '3.9'
             toxenv: py39-test
-
+            
           - name: Python 3.10 with minimal dependencies
             os: ubuntu-latest
             python-version: '3.10'
             toxenv: py310-test
 
           - name: Code style checks
             os: ubuntu-latest
-            python-version: 3.8
+            python-version: 3.9
             toxenv: codestyle
 
     steps:

CHANGES.md

@@ -1,9 +1,22 @@
-1.11.0 (2023-08-10)
+1.13.1 (2023-10-21)
+-------------------
+- bugfix to the docker build
+
+1.13.0 (2023-10-31)
+-------------------
+- Refreshed build enviornment
+- We now push tags to ghcr.io instead of an internal docker registry
+
+1.12.0 (2023-08-10)
 -------------------
 - Added the process_by_group keyword to stages to fix a bug that wouldn't allow grouping only by instrument
 - Updated the logging scheme
 - Ack tasks late so that they may be re-queued in the event of a worker shutdown a la kubernetes
 
+1.11.0 (2023-08-10)
+-------------------
+- Added optional dark temperature scaling
+
 1.10.1 (2023-05-31)
 -------------------
 - We now silently fix fits errors that can be repaired automatically

Dockerfile

@@ -1,26 +1,26 @@
-FROM continuumio/miniconda3:4.10.3
+FROM continuumio/miniconda3:23.5.2-0
 
-RUN conda install -y numpy pip scipy astropy pytest mock requests ipython coverage pyyaml\
-        && conda install -y -c conda-forge kombu=4.4.0 elasticsearch\<6.0.0,\>=5.0.0 pytest-astropy mysql-connector-python\
-        && conda clean -y --all
-
-RUN pip install --no-cache-dir cython logutils lcogt_logging python-dateutil sqlalchemy\>=1.3.0b1 psycopg2-binary celery[redis]==4.3.0 \
-        apscheduler ocs-ingester tenacity amqp==2.6.0 cosmic-conn
+#  In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference
+RUN apt-get -y update && apt-get -y install gcc && \
+        apt-get autoclean && \
+        rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \
         && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \
         && chown -R archive:domainusers /home/archive
 
-COPY --chown=10087:10000 . /lco/banzai
-
-RUN apt-get -y update && apt-get -y install gcc && \
-        pip install --no-cache-dir git+https://github.com/cmccully/sep.git@deblending /lco/banzai/ && \
-        apt-get -y remove gcc && \
-        apt-get autoclean && \
-        rm -rf /var/lib/apt/lists/*
-
 USER archive
 
 ENV HOME /home/archive
 
 WORKDIR /home/archive
+
+COPY environment.yaml .
+
+RUN  . /opt/conda/etc/profile.d/conda.sh && conda config --set remote_read_timeout_secs 900 && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba
+
+COPY --chown=10087:10000 . /lco/banzai
+
+ENV PATH /home/archive/envs/banzai/bin:$PATH
+
+RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ 

banzai/tests/e2e-k8s.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: banzai-e2e-test
-  namespace: build
   labels:
     app.kubernetes.io/name: banzai
 spec:
@@ -62,7 +61,7 @@ spec:
         initialDelaySeconds: 5
         periodSeconds: 1
     - name: banzai-fits-exchange
-      image: rabbitmq:3.7.9
+      image: rabbitmq:3.12.3
       imagePullPolicy: IfNotPresent
       resources:
         requests:
@@ -80,7 +79,7 @@ spec:
         periodSeconds: 1
         timeoutSeconds: 10
     - name: banzai-celery-workers
-      image: @BANZAI_IMAGE@
+      image: banzai:test-latest
       imagePullPolicy: IfNotPresent
       volumeMounts:
         - name: banzai-data
@@ -101,7 +100,7 @@ spec:
         - name: OBSERVATION_PORTAL_URL
           value: "http://internal-observation-portal.lco.gtn/api/observations/"
         - name: OMP_NUM_THREADS
-          value: "4"
+          value: "2"
         - name: FITS_EXCHANGE
           value: "fits_files"
         - name: OPENTSDB_PYTHON_METRICS_TEST_MODE
@@ -118,9 +117,9 @@ spec:
         - --hostname
         - "banzai-celery-worker"
         - --concurrency
-        - "4"
+        - "2"
         - -l
-        - "debug"
+        - "info"
         - "-Q"
         - "$(CELERY_TASK_QUEUE_NAME)"
       readinessProbe:
@@ -144,7 +143,7 @@ spec:
           cpu: 8
           memory: 8Gi
     - name: banzai-celery-beat
-      image: @BANZAI_IMAGE@
+      image: banzai:test-latest
       imagePullPolicy: IfNotPresent
       volumeMounts:
         - name: banzai-data
@@ -170,7 +169,7 @@ spec:
           cpu: 1
           memory: 1Gi
     - name: banzai-listener
-      image: @BANZAI_IMAGE@
+      image: banzai:test-latest
       imagePullPolicy: IfNotPresent
       volumeMounts:
         - name: banzai-data