Merge pull request #1105 from OasisLMF/release/2.3.8

Release 2.3.8
OasisLMF · Sep 20, 2024 · 9cea1c5 · 9cea1c5
2 parents cb316c7 + 6b1b910
commit 9cea1c5
Show file tree

Hide file tree

Showing 23 changed files with 146 additions and 62 deletions.
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -163,7 +163,7 @@ jobs:
 
     - name: Store image size report
       if: success() || failure()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ matrix.dive }}
         path: ${{ matrix.dive }}
@@ -201,7 +201,7 @@ jobs:
 
     - name: Store CVE report
       if: success() || failure() && env.SEVERITY != 'SKIP'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ matrix.report }}
         path: ${{ matrix.report }}

diff --git a/.github/workflows/build-schema.yml b/.github/workflows/build-schema.yml
@@ -5,9 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
-      - stable**
   workflow_dispatch:
     inputs:
       ods_branch:
@@ -49,7 +46,7 @@ jobs:
 
     - name: Download package
       if: needs.ods_tools.outputs.whl_filename != ''
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: bin_package
         path: ${{ github.workspace }}/
@@ -83,21 +80,21 @@ jobs:
         OASIS_GEN_SWAGGER_V2: 1
 
     - name: Store OpenAPI schema
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: openapi-schema
         path: ${{ env.SCHEMA_ALL }}
         retention-days: 3
 
     - name: Store OpenAPI schema (only v1)
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: v1-openapi-schema
         path: ${{ env.SCHEMA_V1 }}
         retention-days: 3
 
     - name: Store OpenAPI schema (only v2)
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: v2-openapi-schema
         path: ${{ env.SCHEMA_V2 }}

diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
@@ -5,7 +5,7 @@
 
 name: Code Quality
 
-on: [push, pull_request]
+on: [pull_request]
 
 env:
   max_line_length: 150

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -330,21 +330,21 @@ jobs:
         docker run --entrypoint "eve" ${{ needs.build_images.outputs.worker_image }} '-v' 2> KTOOLS_VERSION
 
     - name: 'store OASISLMF_VERSION'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: OASISLMF_VERSION
         path: OASISLMF_VERSION
         retention-days: 5
 
     - name: 'store ODS_VERSION'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ODS_VERSION
         path: ODS_VERSION
         retention-days: 5
 
     - name: 'store KTOOLS_VERSION'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: KTOOLS_VERSION
         path: KTOOLS_VERSION
@@ -362,19 +362,19 @@ jobs:
         test "v$KTOOL_VER" = ${{ env.ktools_release }} || exit 1
 
     - name: Download API schema
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: openapi-schema
         path: ${{ github.workspace }}/
 
     - name: Download API schema (v1)
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: v1-openapi-schema
         path: ${{ github.workspace }}/
 
     - name: Download API schema (v2)
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: v2-openapi-schema
         path: ${{ github.workspace }}/

diff --git a/.github/workflows/scan-external.yml b/.github/workflows/scan-external.yml
@@ -95,7 +95,7 @@ jobs:
 
     - name: Store report
       if: success() || failure()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ matrix.report }}
         path: ${{ matrix.report }}

diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml
@@ -5,11 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
-      - stable**  
-  schedule:
-    - cron:  '0 */6 * * *' # Run scan every 6 hours
 
   workflow_dispatch:
     inputs:
@@ -103,7 +98,7 @@ jobs:
 
     - name: Store CVE report
       if: success() || failure()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{ env.REPORT }}
         path: ${{ env.REPORT }}

diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml
@@ -5,9 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
-      - stable**
   workflow_dispatch:
     inputs:
       last_release:

diff --git a/.github/workflows/test-python_api.yml b/.github/workflows/test-python_api.yml
@@ -5,9 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
-      - stable**  
   workflow_dispatch:
     inputs:
       ods_branch:
@@ -59,7 +56,7 @@ jobs:
 
     - name: Download package
       if: needs.ods_tools.outputs.whl_filename != ''
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: bin_package
         path: ${{ github.workspace }}/

diff --git a/.github/workflows/test-python_worker-controller.yml b/.github/workflows/test-python_worker-controller.yml
@@ -5,9 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
-      - stable**  
   workflow_dispatch:
   workflow_call:
 

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,18 @@
 OasisPlatform Changelog
 =======================
 
+`2.3.8`_
+ ---------
+* [#1098](https://github.com/OasisLMF/OasisPlatform/pull/1099) - API is down. Please try again later 
+* [#1075](https://github.com/OasisLMF/OasisPlatform/pull/1103) - Logging - V2 input generation and losses tar should include log files for all sub-tasks 
+* [#1104](https://github.com/OasisLMF/OasisPlatform/pull/1104) - Fix params for V1 workers - so custom hooks are called
+* [#1107](https://github.com/OasisLMF/OasisPlatform/pull/1107) - Fix/gen log storage
+* [#1109](https://github.com/OasisLMF/OasisPlatform/pull/1109) - set artifact to v4
+* [#1079](https://github.com/OasisLMF/OasisPlatform/pull/1079) - Release 2.3.7 (Aug 6) 
+* [#1111](https://github.com/OasisLMF/OasisPlatform/pull/1112) - Keycloak OIDC group permistions is broken. 
+* [#1110](https://github.com/OasisLMF/OasisPlatform/pull/1113) - Add Endpoint for models - 'storage_links'
+.. _`2.3.8`:  https://github.com/OasisLMF/OasisPlatform/compare/2.3.7...2.3.8
+
 `2.3.7`_
  ---------
 * [#1090](https://github.com/OasisLMF/OasisPlatform/pull/1090) -  Fixed CVEs from fiona package (backport)

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.3.7
+2.3.8
diff --git a/kubernetes/charts/oasis-platform/resources/oasis-realm.json b/kubernetes/charts/oasis-platform/resources/oasis-realm.json
@@ -568,8 +568,8 @@
     "authenticationFlowBindingOverrides" : { },
     "fullScopeAllowed" : true,
     "nodeReRegistrationTimeout" : -1,
-    "defaultClientScopes" : [ "web-origins", "acr", "openid", "roles", "profile", "email" ],
-    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+    "defaultClientScopes" : [ "web-origins", "acr", "openid", "roles", "profile", "email", "microprofile-jwt" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access" ]
   }, {
     "id" : "40223c47-11d9-49e5-a552-197fbbfb21c3",
     "clientId" : "realm-management",
@@ -1158,7 +1158,7 @@
       "id" : "21ae480f-ef05-4436-a9d8-a6089272010a",
       "name" : "groups",
       "protocol" : "openid-connect",
-      "protocolMapper" : "oidc-usermodel-realm-role-mapper",
+      "protocolMapper" : "oidc-group-membership-mapper",
       "consentRequired" : false,
       "config" : {
         "multivalued" : "true",

diff --git a/requirements-server.txt b/requirements-server.txt
@@ -207,7 +207,7 @@ numpy==1.25.2
     #   pyarrow
 oasis-data-manager==0.1.3
     # via ods-tools
-ods-tools==3.2.6
+ods-tools==3.2.7
     # via -r requirements-server.in
 packaging==23.0
     # via

diff --git a/requirements-worker.txt b/requirements-worker.txt
@@ -84,8 +84,10 @@ fastparquet==2023.2.0
     #   oasislmf
 filelock==3.10.7
     # via -r requirements-worker.in
-fiona==1.10b2
-    # via geopandas
+fiona==1.9.6
+    # via
+    #   geopandas
+    #   oasislmf
 forex-python==1.8
     # via oasislmf
 fsspec==2023.3.0
@@ -143,9 +145,9 @@ oasis-data-manager==0.1.1
     # via
     #   oasislmf
     #   ods-tools
-oasislmf[extra]==2.3.7
+oasislmf[extra]==2.3.8
     # via -r requirements-worker.in
-ods-tools==3.2.6
+ods-tools==3.2.7
     # via oasislmf
 packaging==23.0
     # via
@@ -222,6 +224,7 @@ six==1.16.0
     #   anytree
     #   azure-core
     #   click-repl
+    #   fiona
     #   isodate
     #   pathlib2
     #   python-dateutil

diff --git a/requirements.txt b/requirements.txt
@@ -217,8 +217,10 @@ filelock==3.10.7
     #   -r ./requirements-worker.in
     #   tox
     #   virtualenv
-fiona==1.10b2
-    # via geopandas
+fiona==1.9.6
+    # via
+    #   geopandas
+    #   oasislmf
 flake8==6.0.0
     # via -r requirements.in
 forex-python==1.8
@@ -334,9 +336,9 @@ oasis-data-manager==0.1.1
     # via
     #   oasislmf
     #   ods-tools
-oasislmf[extra]==2.3.7
+oasislmf[extra]==2.3.8
     # via -r ./requirements-worker.in
-ods-tools==3.2.6
+ods-tools==3.2.7
     # via
     #   -r ./requirements-server.in
     #   oasislmf
@@ -501,6 +503,7 @@ six==1.16.0
     #   automat
     #   azure-core
     #   click-repl
+    #   fiona
     #   isodate
     #   pathlib2
     #   python-dateutil

diff --git a/scripts/deploy.sh b/scripts/deploy.sh
@@ -37,4 +37,4 @@ fi
 set -e
 docker build -f Dockerfile.api_server -t coreoasis/api_server:dev .
 docker build -f Dockerfile.model_worker -t coreoasis/model_worker:dev .
-docker-compose up -d
+docker-compose -f compose/debug.docker-compose.yml up -d
diff --git a/src/model_execution_worker/distributed_tasks.py b/src/model_execution_worker/distributed_tasks.py
@@ -496,7 +496,9 @@ def prepare_input_generation_params(
     gen_files_params = OasisManager()._params_generate_oasis_files(**lookup_params)
     params = paths_to_absolute_paths({**gen_files_params}, config_path)
 
+    params['log_storage'] = dict()
     params['log_location'] = filestore.put(kwargs.get('log_filename'))
+    params['log_storage'][slug] = params['log_location']
     params['verbose'] = debug_worker
     return params
 
@@ -540,6 +542,7 @@ def pre_analysis_hook(self,
     else:
         logger.info('pre_generation_hook: SKIPPING, param "exposure_pre_analysis_module" not set')
     params['log_location'] = filestore.put(kwargs.get('log_filename'))
+    params['log_storage'][slug] = params['log_location']
     return params
 
 
@@ -599,6 +602,7 @@ def prepare_keys_file_chunk(
         )
 
     params['log_location'] = filestore.put(kwargs.get('log_filename'))
+    params['log_storage'][slug] = params['log_location']
     return params
 
 
@@ -690,6 +694,7 @@ def take_first(paths, output_file):
             )
 
     chunk_params['log_location'] = filestore.put(kwargs.get('log_filename'))
+    chunk_params['log_storage'][slug] = chunk_params['log_location']
     return chunk_params
 
 
@@ -714,13 +719,23 @@ def write_input_files(self, params, run_data_uuid=None, analysis_id=None, initia
     if params['user_data_dir'] is not None:
         shutil.rmtree(params['user_data_dir'], ignore_errors=True)
 
+    # collect logs here before archive is created
+    params['log_location'] = filestore.put(kwargs.get('log_filename'))
+    params['log_storage'][slug] = params['log_location']
+    log_dir = os.path.join(params['target_dir'], 'log')
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+
+    for log_ref in params['log_storage']:
+        filestore.get(params['log_storage'][log_ref], os.path.join(log_dir, f'v2-{log_ref}.txt'))
+
     return {
         'lookup_error_location': filestore.put(os.path.join(params['target_dir'], 'keys-errors.csv')),
         'lookup_success_location': filestore.put(os.path.join(params['target_dir'], 'gul_summary_map.csv')),
         'lookup_validation_location': filestore.put(os.path.join(params['target_dir'], 'exposure_summary_report.json')),
         'summary_levels_location': filestore.put(os.path.join(params['target_dir'], 'exposure_summary_levels.json')),
         'output_location': filestore.put(params['target_dir']),
-        'log_location': filestore.put(kwargs.get('log_filename')),
+        'log_location': params['log_location'],
     }
 
 
@@ -931,7 +946,7 @@ def generate_losses_chunk(self, params, chunk_idx, num_chunks, analysis_id=None,
     if num_chunks == 1:
         # Run multiple ktools pipes (based on cpu cores)
         current_chunk_id = None
-        max_chunk_id = -1
+        max_chunk_id = params.get('ktools_num_processes', -1)
         work_dir = 'work'
     else:
         # Run a single ktools pipe
@@ -1000,7 +1015,7 @@ def generate_losses_output(self, params, analysis_id=None, slug=None, **kwargs):
     return {
         **res,
         'output_location': filestore.put(os.path.join(res['model_run_dir'], 'output'), arcname='output'),
-        'run_logs': filestore.put(os.path.join(res['model_run_dir'], 'log'), arcname='logs'),
+        'run_logs': filestore.put(os.path.join(res['model_run_dir'], 'log')),
         'log_location': filestore.put(kwargs.get('log_filename')),
     }