Fix/v2 log storage aws location (#1024)

* Fix S3 example * test sub-task log storage fix * Fix initiator * PEP * Updated Package Requirements: gunicorn==22.0.0 sqlparse==0.5.0 --------- Co-authored-by: awsbuild <[email protected]>
OasisLMF · Apr 23, 2024 · f9e9894 · f9e9894
1 parent 338b85a
commit f9e9894
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 20 deletions.
diff --git a/compose/s3.docker-compose.yml b/compose/s3.docker-compose.yml
@@ -31,6 +31,7 @@ x-oasis-env: &server-env
   OASIS_AWS_S3_ENDPOINT_URL: "http://localstack-s3:4572"
   OASIS_AWS_ACCESS_KEY_ID: foobar
   OASIS_AWS_SECRET_ACCESS_KEY: foobar
+  OASIS_AWS_LOCATION: 'oasis/server'
 
 x-worker-env: &worker-env
   OASIS_DEBUG: 1
@@ -46,12 +47,13 @@ x-worker-env: &worker-env
   OASIS_RABBIT_PORT: 5672
   OASIS_RABBIT_USER: rabbit
   OASIS_RABBIT_PASS: rabbit
-  # S3 
+  # S3
   OASIS_STORAGE_TYPE: S3
   OASIS_AWS_BUCKET_NAME: example-bucket
   OASIS_AWS_S3_ENDPOINT_URL: "http://localstack-s3:4572"
   OASIS_AWS_ACCESS_KEY_ID: foobar
   OASIS_AWS_SECRET_ACCESS_KEY: foobar
+  OASIS_AWS_LOCATION: 'oasis/worker'
 
 services:
   localstack-s3:
@@ -125,20 +127,42 @@ services:
       - broker
     environment:
       <<: *server-env
-  worker:
+  v1-worker:
     restart: always
     image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-dev}
+    build:
+      context: .
+      dockerfile: Dockerfile.model_worker
+    links:
+     - celery-db
+     - broker:mybroker
+    environment:
+     <<: *worker-env
+     OASIS_MODEL_SUPPLIER_ID: OasisLMF
+     OASIS_MODEL_ID: PiWind
+     OASIS_MODEL_VERSION_ID: 'v1'
+     OASIS_RUN_MODE: v1
+    volumes:
+     - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw
+     - ../src/model_execution_worker:/home/worker/src/model_execution_worker
+  v2-worker:
+    restart: always
+    image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-dev}
+    build:
+      context: .
+      dockerfile: Dockerfile.model_worker
     links:
      - celery-db
      - broker:mybroker
     environment:
      <<: *worker-env
      OASIS_MODEL_SUPPLIER_ID: OasisLMF
      OASIS_MODEL_ID: PiWind
-     OASIS_MODEL_VERSION_ID: 1
-     OASIS_RUN_MODE: ${WORKER_API_VER:-v1}
+     OASIS_MODEL_VERSION_ID: 'v2'
+     OASIS_RUN_MODE: v2
     volumes:
-     - ../:/home/worker/model
+     - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw
+     - ../src/model_execution_worker:/home/worker/src/model_execution_worker
   server-db:
     restart: always
     image: postgres

diff --git a/requirements-server.txt b/requirements-server.txt
@@ -142,7 +142,7 @@ fsspec==2024.2.0
     #   oasis-data-manager
 greenlet==2.0.2
     # via sqlalchemy
-gunicorn==20.1.0
+gunicorn==22.0.0
     # via -r requirements-server.in
 hiredis==2.2.2
     # via aioredis
@@ -217,6 +217,7 @@ packaging==23.0
     # via
     #   drf-yasg
     #   fastparquet
+    #   gunicorn
     #   ods-tools
 pandas==2.1.4
     # via
@@ -293,7 +294,7 @@ six==1.16.0
     #   service-identity
 sqlalchemy==1.4.47
     # via -r requirements-server.in
-sqlparse==0.4.4
+sqlparse==0.5.0
     # via
     #   django
     #   django-debug-toolbar

diff --git a/requirements.txt b/requirements.txt
@@ -233,7 +233,7 @@ geopandas==0.12.2
     # via oasislmf
 greenlet==2.0.2
     # via sqlalchemy
-gunicorn==20.1.0
+gunicorn==22.0.0
     # via -r ./requirements-server.in
 hiredis==2.2.2
     # via aioredis
@@ -352,6 +352,7 @@ packaging==23.0
     #   drf-yasg
     #   fastparquet
     #   geopandas
+    #   gunicorn
     #   ods-tools
     #   pyproject-api
     #   pytest
@@ -521,7 +522,7 @@ sqlalchemy==1.4.47
     # via
     #   -r ./requirements-server.in
     #   -r ./requirements-worker.in
-sqlparse==0.4.4
+sqlparse==0.5.0
     # via
     #   django
     #   django-debug-toolbar

diff --git a/src/server/oasisapi/analyses/v2_api/tasks.py b/src/server/oasisapi/analyses/v2_api/tasks.py
@@ -493,7 +493,7 @@ def record_input_files(self, result, analysis_id=None, initiator_id=None, run_da
     # group sub-task logs and write to trace file
     random_filename = '{}.txt'.format(uuid.uuid4().hex)
     with TemporaryFile() as tmp_file:
-        # Write Error logs
+       # Write Error logs
         subtask_qs = analysis.sub_task_statuses.filter(
             status__in=[AnalysisTaskStatus.status_choices.COMPLETED]
         )
@@ -583,6 +583,7 @@ def record_sub_task_start(self, analysis_id=None, task_slug=None, task_id=None,
 def record_sub_task_success(self, res, analysis_id=None, initiator_id=None, task_slug=None):
     log_location = res.get('log_location')
     error_location = res.get('error_location')
+    initiator = get_user_model().objects.get(pk=initiator_id)
 
     task_id = self.request.parent_id
     AnalysisTaskStatus.objects.filter(
@@ -592,17 +593,11 @@ def record_sub_task_success(self, res, analysis_id=None, initiator_id=None, task
         task_id=task_id,
         status=AnalysisTaskStatus.status_choices.COMPLETED,
         end_time=timezone.now(),
-        output_log=None if not log_location else RelatedFile.objects.create(
-            file=str(log_location),
-            filename='{}-output.log'.format(task_id),
-            content_type='text/plain',
-            creator_id=initiator_id,
+        output_log=None if not log_location else store_file(
+            log_location, 'text/plain', initiator, filename=f'{task_id}-output.log'
         ),
-        error_log=None if not error_location else RelatedFile.objects.create(
-            file=str(error_location),
-            filename='{}-error.log'.format(task_id),
-            content_type='text/plain',
-            creator_id=initiator_id,
+        error_log=None if not error_location else store_file(
+            error_location, 'text/plain', initiator, filename=f'{task_id}-error.log'
         )
     )