biosimulators · AlexPatrie · Oct 30, 2024 · Oct 30, 2024 · Oct 30, 2024 · Oct 30, 2024
diff --git a/.github/workflows/cd.yaml → .github/workflows/deploy.yml b/.github/workflows/cd.yaml → .github/workflows/deploy.yml
@@ -1,65 +1,53 @@
-name: CD
+name: Deploy
 
+# on:
+#   workflow_dispatch:
 on:
-  workflow_dispatch:
+  workflow_run:
+    workflows: ["Test"]
+    types:
+      - completed
 
 jobs:
-  configure-environment:
+  deploy-base-image:
     runs-on: ubuntu-latest
     steps:
-      # Step 1: Checkout repository
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-      # Step 2: Log in to GHCR
       - name: Log in to GHCR
         run: echo "${{ secrets.REPO_ADMIN_GHCR_TOKEN }}" | docker login ghcr.io -u "${{ secrets.REPO_ADMIN_GH_USERNAME }}" --password-stdin
 
-      # Step 3: Install Docker Compose
-      - name: Install Docker Compose
-        run: |
-          sudo curl -L "https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4)/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          sudo chmod +x /usr/local/bin/docker-compose
-          docker-compose --version  # Check if Docker Compose is installed correctly
-
-      - name: Install Python and PyYAML
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3 python3-pip
-          pip install pyyaml
-
-  build-base-image:
-    needs: configure-environment
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Get base image version
+      - name: Get base image tag and version
         run: |
           BASE_VERSION=$(cat ./assets/.BASE_VERSION)
           echo "BASE_VERSION=$BASE_VERSION" >> $GITHUB_ENV
+          BASE_IMG=ghcr.io/biosimulators/bio-check-base
+          echo "BASE_IMG=$BASE_IMG" >> $GITHUB_ENV
           echo "Base Version: $BASE_VERSION"
+          echo "Base Tag: $BASE_IMG"
 
       - name: Create gcloud config
         run: |
-          echo "$BIOSIMULATIONS_GCLOUD_CONFIG" > ./assets/.biosimulations.json
+          echo "$BIOSIMULATIONS_GCLOUD_CONFIG" > ./assets/configs/.biosimulations.json
         env:
           BIOSIMULATIONS_GCLOUD_CONFIG: ${{ secrets.BIO_JSON_CONTENT }}
 
       - name: Build base image
         run: |
-          sudo chmod +x ./assets/scripts/build_base.sh
-          ./assets/scripts/build_base.sh ${{ env.BASE_VERSION }}
+          docker build --no-cache -f ./Dockerfile-base -t ${{ env.BASE_IMG }}:${{ env.BASE_VERSION }} .
+
+      - name: Deploy new base version to GHCR
+        run: |
+          docker push ${{ env.BASE_IMG }}:${{ env.BASE_VERSION }}
 
-      - name: Push base image to GHCR
+      - name: Deploy new latest base version
         run: |
-          sudo chmod +x ./assets/scripts/push_base.sh
-          sudo chmod +x ./assets/scripts/push_image.sh
-          ./assets/scripts/push_base.sh ${{ env.BASE_VERSION }} ${{ secrets.REPO_ADMIN_GH_USERNAME }}
+          docker tag ${{ env.BASE_IMG }}:${{ env.BASE_VERSION }} ${{ env.BASE_IMG }}:latest
+          docker push ${{ env.BASE_IMG }}:latest
 
   get-microservice-versions:
-    needs: build-base-image
+    needs: deploy-base-image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -94,37 +82,55 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      # Step 8: Set up Docker Buildx
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      # Step 9: Build Docker containers with Docker Compose
-      - name: Build Docker containers with Docker Compose
-        run: docker-compose build --no-cache
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
 
-  deploy-microservice-images:
-    needs: build-microservice-images
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r assets/configs/requirements.github.txt
+
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4)/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version  # Check if Docker Compose is installed correctly
+
+      - name: Build Docker containers with Docker Compose
+        run: docker-compose build
 
-      # Step 10: Change permissions for push script
-      - name: Change push script permissions
+      - name: Authorize push script
         run: chmod +x ./assets/scripts/push_image.sh
 
-      # Step 11: Deploy API microservice container to GHCR
       - name: Deploy API microservice container to GHCR
         run: |
-          ./assets/scripts/push_image.sh compose_api ${{ env.API_VERSION }}
+          rm api/spec/openapi_3_1_0_generated.yaml
+          python3 api/openapi_spec.py
+          sudo rm -r api/__pycache__
+          ./assets/scripts/push_image.sh api ${{ env.API_VERSION }}
+        # STABLE CONTENT:
+        # - run: |
+        #   rm compose_api/spec/openapi_3_1_0_generated.yaml
+        #   python3 compose_api/openapi_spec.py
+        #   sudo rm -r compose_api/__pycache__
+        #   ./assets/scripts/push_image.sh compose_api ${{ env.API_VERSION }}
         env:
           REPO_ADMIN_GH_USERNAME: ${{ secrets.REPO_ADMIN_GH_USERNAME }}
           REPO_ADMIN_GHCR_TOKEN: ${{ secrets.REPO_ADMIN_GHCR_TOKEN }}
 
-      # Step 12: Deploy Worker microservice container to GHCR
       - name: Deploy worker microservice container to GHCR
         run: |
-          ./assets/scripts/push_image.sh compose_worker ${{ env.WORKER_VERSION }}
+          sudo rm -r compose_api/__pycache__
+          ./assets/scripts/push_image.sh worker ${{ env.WORKER_VERSION }}
+        # STABLE CONTENT:
+        # run: |
+        #   sudo rm -r compose_api/__pycache__
+        #   ./assets/scripts/push_image.sh compose_worker ${{ env.WORKER_VERSION }}
         env:
           REPO_ADMIN_GH_USERNAME: ${{ secrets.REPO_ADMIN_GH_USERNAME }}
-          REPO_ADMIN_GHCR_TOKEN: ${{ secrets.REPO_ADMIN_GHCR_TOKEN }}
+          REPO_ADMIN_GHCR_TOKEN: ${{ secrets.REPO_ADMIN_GHCR_TOKEN }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,28 @@
+name: Test
+
+# TODO: add main here
+on:
+  push:
+
+jobs:
+  build-base-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Get new base image version
+        run: |
+          BASE_VERSION=$(cat ./assets/.BASE_VERSION)
+          echo "BASE_VERSION=$BASE_VERSION" >> $GITHUB_ENV
+          echo "Base Version: $BASE_VERSION"
+
+      - name: Create gcloud config artifact
+        run: |
+          echo "$BIOSIMULATIONS_GCLOUD_CONFIG" > ./assets/configs/.biosimulations.json
+        env:
+          BIOSIMULATIONS_GCLOUD_CONFIG: ${{ secrets.BIO_JSON_CONTENT }}
+
+      - name: Test build base image
+        run: |
+          docker build --no-cache -f ./Dockerfile-base -t ghcr.io/biosimulations/bio-check-base:${{ env.BASE_VERSION }} .
diff --git a/.gitignore b/.gitignore
@@ -11,9 +11,9 @@ venv
 smoldyn-2.73-mac.tgz.1
 *.egg-info/
 assets/dev/.env_dev
-assets/.bio-check.json
-assets/.biosimulations-prodbucket.json
-assets/.biosimulations.json
+assets/configs/.bio-check.json
+assets/configs/.biosimulations-prodbucket.json
+assets/configs/.biosimulations.json
 bucket.sh
 .env_dev
 NOTES

diff --git a/Dockerfile-base b/Dockerfile-base
@@ -1,3 +1,5 @@
+# Base image for app microservices. IMPORTANT: Current stable version in production is 0.1.5
+
 # FROM ghcr.io/biosimulators/biosimulators:latest THE STABLE VERSION OF BIOSIMULATOR PROCESSES IMAGE IS 0.0.4
 # FROM ghcr.io/vivarium-collective/biosimulator-processes:0.0.4
 
@@ -10,7 +12,7 @@ ENV MONGO_URI="mongodb://mongodb/?retryWrites=true&w=majority&appName=bio-check"
 
 # handle app creds from google
 RUN mkdir /.google
-COPY ./assets/.biosimulations.json /.google/.bio-check.json
+COPY assets/configs/.biosimulations.json /.google/.bio-check.json
 # COPY ./.bio-check.json /.google/.bio-check.json
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -51,12 +53,13 @@ WORKDIR /app
 RUN mkdir /app/data
 
 # copy assets
-COPY ./assets/requirements.base.txt ./assets/scripts/remove_deps.sh ./assets/scripts/update_deps.sh ./assets/dropped.txt ./assets/scripts/install_deps.sh /app/assets/
-COPY pyproject.toml poetry.lock /app/
+COPY assets/configs/requirements.base.txt ./assets/scripts/remove_deps.sh ./assets/scripts/update_deps.sh ./assets/dropped.txt ./assets/scripts/install_deps.sh /app/assets/
+COPY pyproject.toml /app/pyproject.toml
+# COPY pyproject.toml poetry.lock /app/
 
 # Configure PySCeS from assets
-COPY ./assets/.pys_usercfg.ini /Pysces/.pys_usercfg.ini
-COPY ./assets/.pys_usercfg.ini /root/Pysces/.pys_usercfg.ini
+COPY assets/configs/.pys_usercfg.ini /Pysces/.pys_usercfg.ini
+COPY assets/configs/.pys_usercfg.ini /root/Pysces/.pys_usercfg.ini
 RUN mkdir -p /Pysces \
     && mkdir -p /Pysces/psc \
     && mkdir -p /root/Pysces \
@@ -70,52 +73,11 @@ COPY ./model-examples/sbml-core/Elowitz-Nature-2000-Repressilator /app/data/Elow
 # create readme for poetry
 RUN echo "BioCompose Server" > /app/README.md
 
-# install poetry and deps
+# install poetry deps
 RUN python3.10 -m pip install --upgrade pip \
     && python3.10 -m pip install poetry \
     && poetry config virtualenvs.in-project true \
     && poetry env use 3.10 \
-    && poetry install --without=api,worker,pysces,masspy
-
-
-# install smoldyn
-# RUN . /app/.venv/bin/activate \
-#     && wget https://www.smoldyn.org/smoldyn-2.73.tgz \
-#     && tar -xzf smoldyn-2.73.tgz \
-#     && cd smoldyn-2.73 \
-#     && cd build \
-#     && cmake .. -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=$(which python3.10) \
-#     && make \
-#     && make install
-# && rm -rf /var/lib/apt/lists/* \
-# && apt-get clean \
-# && apt-get autoclean
-
-# RUN . /app/.venv/bin/activate \
-#     && curl -O https://download.opensuse.org/repositories/home:/dilawar/xUbuntu_22.04/amd64/smoldyn_2.64.4-1+2.1_amd64.deb \
-#     && dpkg -i smoldyn_2.64.4-1+2.1_amd64.deb
-
-# RUN . /app/.venv/bin/activate \
-#     && echo 'deb http://download.opensuse.org/repositories/home:/dilawar/xUbuntu_22.04/ /' | tee /etc/apt/sources.list.d/home:dilawar.list \
-#     && curl -fsSL https://download.opensuse.org/repositories/home:dilawar/xUbuntu_22.04/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/home_dilawar.gpg > /dev/null \
-#     && apt update \
-#     && apt install smoldyn
-
-# fix python version constraint, lock env, and check compatible
-# RUN sed -i 's/python = ">=3.10"/python = "^3.10"/' /app/pyproject.toml \
-#     && poetry lock \
-#     && poetry check
-
-# TODO: fix this eventually, but for now (10/2024) biosimulators-processes is not used
-# RUN rm -r biosimulator_processes
-
-# remove existing cobra installation TODO: clean this up
-# RUN poetry remove cobra && poetry remove zarr
-
-# install project deps (shared)
-# RUN chmod +x /app/assets/install_deps.sh \
-#     && /app/assets/install_deps.sh /app/assets/requirements.base.txt
-
+    && poetry lock \
+    && poetry install --without=api,worker
 
-# TODO: create shared tooling module which is a set of the sum of api and worker data_model.py & shared.py
-# COPY ./assets/shared.py /app/shared.py
diff --git a/_Dockerfile-base → ...ile-base_BIOSIMULATOR_PROCESSES(original) b/_Dockerfile-base → ...ile-base_BIOSIMULATOR_PROCESSES(original)