Update CI testing file (#164)

OasisLMF · Nov 8, 2023 · 306cc52 · 306cc52
1 parent 66544d2
commit 306cc52
Show file tree

Hide file tree

Showing 4 changed files with 188 additions and 22 deletions.
diff --git a/docker/Dockerfile.piwind_server b/docker/Dockerfile.piwind_server
@@ -0,0 +1,21 @@
+FROM coreoasis/api_server:latest
+
+COPY --chown=server:server $ods_package ./
+USER server
+
+# Install ods_tools from package file (Optional) 'docker build --build-arg ods_package=<whl>'
+ARG ods_package
+RUN if [ ! -z "$ods_package" ];  then \
+    pip uninstall ods-tools -y; \
+    pip install --user --no-warn-script-location $ods_package; \
+  fi
+
+# Install ODS-Tools from git branch (Optional) 'docker build --build-arg ods_tools_branch=develop'
+ARG ods_tools_branch
+RUN if [ ! -z "$ods_tools_branch" ] ; then \
+    apt update && apt install -y git; \
+    pip uninstall ods-tools -y; \
+    pip install --user --no-warn-script-location -v git+https://[email protected]/OasisLMF/ODS_Tools.git@${ods_tools_branch}#egg=ods-tools; \
+  fi
+
+
diff --git a/docker/Dockerfile.piwind_worker b/docker/Dockerfile.piwind_worker
@@ -1,41 +1,38 @@
 FROM coreoasis/model_worker:latest
 
-
-# Install MDK from git branch
-#    'docker build --build-arg oasislmf_branch=develop ...'
-ARG oasislmf_branch
-
-# Instal local MDK package in worker
-#    'docker build --build-arg oasislmf_package=oasislmf-1.27.0rc3-py3-none-manylinux1_x86_64.whl ...'
-ARG oasislmf_package
-
-# Instal local ods_tools package
-#    'docker build --build-arg ods_tools_package=<package-name>.whl ...'
-ARG ods_package
-
-
 WORKDIR /home/worker
 COPY ./docker/oasislmf_deploy.json /home/worker/model/oasislmf.json
 COPY ./meta-data/model_settings.json /home/worker/model/meta-data/model_settings.json
 ENTRYPOINT ./startup.sh
 
-# Copy in packages if given
 COPY $ods_package ./
 COPY $oasislmf_package ./
 
-# Install ods_tools from package file (Optional)
+
+# Install ods_tools from package file (Optional) 'docker build --build-arg ods_package=<whl>'
+ARG ods_package
 RUN if [ ! -z "$ods_package" ];  then \
     pip uninstall ods-tools -y; \
     pip install $ods_package; \
-  fi    
+  fi
 
-# Install MDK from package file (Optional)
-RUN if [ ! -z "$oasislmf_package" ]; then \ 
+# Install ODS-Tools from git branch (Optional) 'docker build --build-arg ods_tools_branch=develop'
+ARG ods_tools_branch
+RUN if [ ! -z "$ods_tools_branch" ] ; then \
+    apt update && apt install -y git; \
+    pip uninstall ods-tools -y; \
+    pip install --user --no-warn-script-location -v git+https://[email protected]/OasisLMF/ODS_Tools.git@${ods_tools_branch}#egg=ods-tools; \
+  fi
+
+# Install MDK from package file (Optional) 'docker build --build-arg oasislmf_package=<whl>'
+ARG oasislmf_package
+RUN if [ ! -z "$oasislmf_package" ]; then \
     pip uninstall oasislmf -y; \
     pip install $oasislmf_package; \
-  fi 
+  fi
 
-# install MDK from git branch (Optional)
+# install MDK from git branch (Optional) 'docker build --build-arg oasislmf_branch=develop'
+ARG oasislmf_branch
 RUN if [ ! -z "$oasislmf_branch" ] ; then \
     apt update && apt install -y git; \
     pip uninstall oasislmf -y; \

diff --git a/docker/plat2.docker-compose.yml b/docker/plat2.docker-compose.yml
@@ -0,0 +1,144 @@
+version: '3.4'
+volumes:
+  server-db-OasisData:
+  celery-db-OasisData:
+  filestore-OasisData:
+x-shared-env: &shared-env
+  OASIS_DEBUG: 1
+  OASIS_URL_SUB_PATH: 0
+  OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672"
+  OASIS_SERVER_DB_HOST: server-db
+  OASIS_SERVER_DB_PASS: oasis
+  OASIS_SERVER_DB_USER: oasis
+  OASIS_SERVER_DB_NAME: oasis
+  OASIS_SERVER_DB_PORT: 5432
+  OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer
+  OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2
+  OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2
+  OASIS_CELERY_DB_HOST: celery-db
+  OASIS_CELERY_DB_PASS: password
+  OASIS_CELERY_DB_USER: celery
+  OASIS_CELERY_DB_NAME: celery
+  OASIS_CELERY_DB_PORT: 5432
+  OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller
+  OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller
+x-volumes: &shared-volumes
+  - filestore-OasisData:/shared-fs:rw
+services:
+  server:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: ["./wsgi/run-wsgi.sh"]
+   ports:
+     - 8000:8000
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *shared-env
+     STARTUP_RUN_MIGRATIONS: "true"
+     OASIS_ADMIN_USER: admin
+     OASIS_ADMIN_PASS: password
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  websocket:
+   restart: always
+   image: coreoasis/api_server:dev
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: ["./asgi/run-asgi.sh"]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   ports:
+     - 8001:8001
+   environment:
+     <<: *shared-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  worker-monitor:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app, worker, --loglevel=INFO]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *shared-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  task-controller:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app, worker, --loglevel=INFO, -Q, task-controller]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *shared-env
+   volumes:
+     - filestore-OasisData:/shared-fs:rw
+  celery-beat:
+   restart: always
+   image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev}
+   command: [celery, -A, src.server.oasisapi.celery_app, beat, --loglevel=INFO]
+   links:
+     - server-db
+     - celery-db
+     - broker
+   environment:
+     <<: *shared-env
+   volumes: *shared-volumes
+  worker:
+    restart: always
+    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-2.1.0-dev}
+    links:
+     - celery-db
+     - broker:mybroker
+    environment:
+     <<: *shared-env
+     OASIS_MODEL_SUPPLIER_ID: OasisLMF
+     OASIS_MODEL_ID: PiWind
+     OASIS_MODEL_VERSION_ID: 1
+    volumes:
+     - ../:/home/worker/model
+     - filestore-OasisData:/shared-fs:rw
+  server-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=oasis
+      - POSTGRES_USER=oasis
+      - POSTGRES_PASSWORD=oasis
+    volumes:
+      - server-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33307:3306
+  celery-db:
+    restart: always
+    image: postgres
+    environment:
+      - POSTGRES_DB=celery
+      - POSTGRES_USER=celery
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - celery-db-OasisData:/var/lib/postgresql/data:rw
+    ports:
+      - 33306:5432
+  broker:
+    restart: always
+    image: rabbitmq:3.8.14-management
+    environment:
+      - RABBITMQ_DEFAULT_USER=rabbit
+      - RABBITMQ_DEFAULT_PASS=rabbit
+    ports:
+      - 5672:5672
+      - 15672:15672
+  channel-layer:
+    restart: always
+    image: redis:5.0.7
+    ports:
+      - 6379:6379
diff --git a/tests/modelcheck.py b/tests/modelcheck.py
@@ -87,7 +87,11 @@ def wait_for_api(module_scoped_container_getter):
     # Wait for Model
     oasis_client = APIClient(api_url=api_url)
     oasis_client.api.mount('http://', HTTPAdapter(max_retries=retries))
-    assert oasis_client.models.get(1)
+
+    model_headers = {'authorization': f"Bearer {oasis_client.api.tkn_access}"}
+    model_url = f"{api_url}/V1/models/1/"
+    assert request_session.get(model_url, headers=model_headers)
+    #assert oasis_client.models.get(1)
     return oasis_client