Logging fixes for workers (#994) - 1.27 (#998)

* Logging fixes for workers  (#994)

* first pass - refactor input gen call and logging

* Fix exceptions not logged by worker monitor

* remove test exception

* Call generate losses from python instead of bash

* fix logger calls

* wip

* test refactor

* tmp

* Add sub-task error logs to main analysis error trace

* concat output logs on input gen success

* Fix v2 worker gen files

* Fix losses gen traceback

* Fix V1 worker logging

* Fix CI docker log storage

* Fix

* Fix tests

* flak8

* Fix debug logging level

* wip - fix missing ktools log tar

* Fix missing ktools log tar

* pep

* Clean up task log files on exit

* Fix traceback filenames v2

* Enable tests on release branches

* Fix piwind branch

* Add missing tasks log dir

.github/workflows/build-schema.yml

@@ -5,10 +5,12 @@ on:
     branches:
       - main-platform1
       - stable**
+      - release**
   pull_request:
     branches:
       - main-platform1
       - stable**
+      - release**
   workflow_dispatch:
     inputs:
       ods_branch:

.github/workflows/scan.yml

@@ -5,10 +5,12 @@ on:
     branches:
       - main-platform1
       - stable**
+      - release**
   pull_request:
     branches:
       - main-platform1
       - stable**
+      - release**
   schedule:
     - cron:  '0 */6 * * *' # Run scan every 6 hours
 

.github/workflows/test-images.yml

@@ -5,10 +5,12 @@ on:
     branches:
       - main-platform1
       - stable**
+      - release**
   pull_request:
     branches:
       - main-platform1
       - stable**
+      - release**
   workflow_dispatch:
     inputs:
       last_release:
@@ -86,22 +88,13 @@ jobs:
       - name: Select PiWind branch
         id: piwind
         run: |
-          # Select matching base branch on piwind
-          if [[ "${{ github.event_name }}" = "pull_request" ]]; then
-            BRANCH=${{ github.base_ref }}
-          elif [[ "${{ github.event_name }}" = "push" ]]; then
-            BRANCH=${{ github.ref_name }}
+          if [[ -z "${{ inputs.piwind_branch }}" ]]; then
+            BRANCH='stable/1.27.x'
           else
             BRANCH=${{ inputs.piwind_branch }}
           fi
-
-          #override 'main-platform1' -> 'main'
-          if [[ "$BRANCH" = 'main-platform1' ]]; then
-             BRANCH=main
-          fi
           echo "branch=$BRANCH" >> $GITHUB_OUTPUT
 
-
       - name: Select Pytest Options
         id: pytest
         run: |

.github/workflows/test-python.yml

@@ -5,10 +5,13 @@ on:
     branches:
       - main-platform1
       - stable**
+      - release**
   pull_request:
     branches:
       - main-platform1
       - stable**
+      - release**
+
   workflow_dispatch:
     inputs:
       ods_branch:

Dockerfile.model_worker

@@ -53,7 +53,7 @@ COPY ./tests/integration /home/worker/tests/integration
 # Add required directories
 RUN mkdir -p /var/oasis && \
     mkdir -p /home/worker/model && \
-    mkdir -p /var/log/oasis && \
+    mkdir -p /var/log/oasis/tasks && \
     mkdir -p /shared-fs && \
     touch /var/log/oasis/worker.log && \
     chmod 777 /var/log/oasis/worker.log

Dockerfile.model_worker_debian

@@ -40,7 +40,7 @@ COPY ./VERSION ./
 
 RUN mkdir -p /var/oasis && \
     mkdir -p /home/worker/model && \
-    mkdir -p /var/log/oasis && \
+    mkdir -p /var/log/oasis/tasks && \
     mkdir -p /shared-fs && \
     touch /var/log/oasis/worker.log && \
     chmod 777 /var/log/oasis/worker.log