Merge pull request from OasisLMF/release/3.2.0

Release 3.2.0
OasisLMF · Feb 6, 2024 · 8e0a116 · 8e0a116
2 parents 3c9a3a2 + 5b031ca
commit 8e0a116
Show file tree

Hide file tree

Showing 13 changed files with 377 additions and 64 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -168,24 +168,6 @@ jobs:
         name: bin_package
         path: ${{ github.workspace }}/
 
-    - name: Import GPG key
-      id: import_gpg
-      uses: crazy-max/ghaction-import-gpg@v5
-      with:
-        gpg_private_key: ${{ secrets.GPG_PYPI_SIGNING_KEY }}
-        passphrase: ${{ secrets.GPG_PYPI_SIGNING_PASS }}
-
-    - name: GPG sign Source package
-      working-directory: ${{ github.workspace }}
-      run: |
-        gpg --batch --no-tty --passphrase-fd 0 --detach-sign -a ${{ needs.package.outputs.src_filename }}
-
-    - name: GPG sign Wheel package
-      working-directory: ${{ github.workspace }}
-      run: |
-        gpg --batch --no-tty --passphrase-fd 0 --detach-sign -a ${{ needs.package.outputs.whl_filename }}
-
-
     # --- Create Release --- #
     - name: Push changes
       run: |
@@ -204,7 +186,6 @@ jobs:
         draft: false
         prerelease: ${{ env.PRE_RELEASE }}
 
-
     # --- Attach build assest --- #
     - name: Upload Source package
       id: upload-source-release-asset
@@ -234,19 +215,13 @@ jobs:
       run: pip install twine
 
     - name: PYPI - Source package
-      run: |
-        twine upload \
-        ${{ needs.package.outputs.src_filename }} \
-        ${{ needs.package.outputs.src_filename }}.asc
+      run: twine upload ${{ needs.package.outputs.src_filename }}
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
 
     - name: PYPI - Linux package
-      run: |
-        twine upload \
-        ${{ needs.package.outputs.whl_filename }} \
-        ${{ needs.package.outputs.whl_filename }}.asc
+      run: twine upload ${{ needs.package.outputs.whl_filename }}
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

diff --git a/.github/workflows/test-oasislmf.yml b/.github/workflows/test-oasislmf.yml
@@ -9,7 +9,7 @@ on:
         required: false
 
 env:
-  oasislmf_branch: ${{ github.event_name != 'workflow_dispatch' && 'stable/1.28.x' ||  inputs.oasislmf_branch }}
+  oasislmf_branch: ${{ github.event_name != 'workflow_dispatch' && 'main' ||  inputs.oasislmf_branch }}
 
 jobs:
   build_ods:

diff --git a/.github/workflows/test-platform.yml b/.github/workflows/test-platform.yml
@@ -10,8 +10,8 @@ on:
 
 jobs:
   platform_testing:
-    uses: OasisLMF/OasisPlatform/.github/workflows/test-python.yml@main
+    uses: OasisLMF/OasisPlatform/.github/workflows/test-python_api.yml@main
     secrets: inherit
     with:
       ods_branch: ${{ github.ref }}
-      platform_branch: ${{ github.event_name != 'workflow_dispatch' && 'main-platform1' ||  inputs.platform_branch }}
+      platform_branch: ${{ github.event_name != 'workflow_dispatch' && 'main' ||  inputs.platform_branch }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -9,16 +9,22 @@ on:
         required: false
 
 jobs:
-  build:
+  build-ods:
     uses: ./.github/workflows/build.yml
     secrets: inherit
     with:
       oed_spec_branch: ${{ inputs.oed_spec_branch }}
 
+  build-odm:
+    uses: OasisLMF/OasisDataManager/.github/workflows/build.yml@main
+    secrets: inherit
+    with:
+      odm_branch: ${{ github.event_name != 'workflow_dispatch' && 'develop' ||  inputs.ods_branch }}
+
   test:
     name: Run Pytest
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build-ods, build-odm]
     strategy:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
@@ -31,17 +37,25 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Download package
+    - name: Download package (ODS-tools)
       uses: actions/download-artifact@v3
       with:
         name: bin_package
         path: ${{ github.workspace }}/
 
+    - name: Download package (OasisDataManager)
+      uses: actions/download-artifact@v3
+      with:
+        name: odm_bin_package
+        path: ${{ github.workspace }}/
+
     - name: install package
-      run: pip install ${{ needs.build.outputs.whl_filename }}
+      run: |
+        pip install ${{ needs.build-odm.outputs.whl_filename }}
+        pip install ${{ needs.build-ods.outputs.whl_filename }}
 
     - name: Install test deps
       run: pip install -r tests/requirements.in
 
     - name: Run tests
-      run: pytest -v
+      run: pytest -v
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,29 @@
 ODS_Tools Changelog
 ===================
 
+`3.2.0`_
+ ---------
+* [#78](https://github.com/OasisLMF/ODS_Tools/pull/78) - Release 3.1.4
+* [#80](https://github.com/OasisLMF/ODS_Tools/pull/80) - Added fields to analysis settings schema
+* [#81](https://github.com/OasisLMF/ODS_Tools/pull/81) - Add vulnerability adjustments field to schema
+* [#82](https://github.com/OasisLMF/ODS_Tools/pull/83) - Enable additional properties for additional assets in model_settings.json
+* [#72, #55](https://github.com/OasisLMF/ODS_Tools/pull/84) - Remove warning message
+* [#85](https://github.com/OasisLMF/ODS_Tools/pull/86) - Add api run_mode, model running on (v1 / v2), to model settings 
+* [#88](https://github.com/OasisLMF/ODS_Tools/pull/88) - Lot3 integration with ODS-tools package
+* [#89](https://github.com/OasisLMF/ODS_Tools/pull/89) - Fix CI workflow default branches
+.. _`3.2.0`:  https://github.com/OasisLMF/ODS_Tools/compare/3.1.4...3.2.0
+
+`3.2.0`_
+ ---------
+* [#78](https://github.com/OasisLMF/ODS_Tools/pull/78) - Release 3.1.4
+* [#80](https://github.com/OasisLMF/ODS_Tools/pull/80) - Added fields to analysis settings schema
+* [#81](https://github.com/OasisLMF/ODS_Tools/pull/81) - Add vulnerability adjustments field to schema
+* [#82](https://github.com/OasisLMF/ODS_Tools/pull/83) - Enable additional properties for additional assets in model_settings.json
+* [#72, #55](https://github.com/OasisLMF/ODS_Tools/pull/84) - Remove warning message
+* [#85](https://github.com/OasisLMF/ODS_Tools/pull/86) - Add api run_mode, model running on (v1 / v2), to model settings 
+* [#88](https://github.com/OasisLMF/ODS_Tools/pull/88) - Lot3 integration with ODS-tools package
+.. _`3.2.0`:  https://github.com/OasisLMF/ODS_Tools/compare/3.1.4...3.2.0
+
 .. _`3.1.4`:  https://github.com/OasisLMF/ODS_Tools/compare/3.1.3...3.1.4
 
 `3.1.3`_

diff --git a/ods_tools/__init__.py b/ods_tools/__init__.py
@@ -1,4 +1,4 @@
-__version__ = '3.1.4'
+__version__ = '3.2.0'
 
 import logging
 

diff --git a/ods_tools/data/analysis_settings_schema.json b/ods_tools/data/analysis_settings_schema.json
@@ -412,6 +412,52 @@
                 }
             }
         },
+        "vulnerability_adjustments": {
+            "type": "object",
+            "title": "Vulnerability Adjustments",
+            "description": "Object containing vulnerability adjustments and replacements.",
+            "properties": {
+                "adjustments": {
+                    "type": "object",
+                    "title": "Vulnerability Factor Adjustments",
+                    "description": "Dictionary containing vulnerability adjustments with vulnerability ID as integer keys and adjustment factors as float values.",
+                    "patternProperties": {
+                        "^[0-9]+$": {
+                            "type": "number",
+                            "minimum": 0
+                        }
+                    },
+                    "additionalProperties": false
+                },
+                "replace_data": {
+                    "type": "object",
+                    "title": "Vulnerability Replacements",
+                    "description": "Detailed vulnerability replacements.",
+                    "patternProperties": {
+                        "^[0-9]+$": {
+                            "type": "array",
+                            "items": {
+                                "type": "array",
+                                "minItems": 3,
+                                "maxItems": 3,
+                                "items": [
+                                    { "type": "integer" },
+                                    { "type": "integer" },
+                                    { "type": "number", "minimum": 0, "maximum": 1 }
+                                ]
+                            }
+                        }
+                    },
+                    "additionalProperties": false
+                },
+                "replace_file": {
+                    "type": "string",
+                    "title": "Vulnerability Replacement File",
+                    "description": "Path to a CSV file containing vulnerability data to be replaced."
+                }
+            },
+            "additionalProperties": false
+        },    
         "gul_output": {
             "type": "boolean",
             "title": "Produce GUL output",

diff --git a/ods_tools/data/model_settings_schema.json b/ods_tools/data/model_settings_schema.json
@@ -40,6 +40,15 @@
          "description":"The version ID for the model. (optional)",
          "minLength":1
       },
+      "model_run_mode": {
+         "type":"string",
+         "title":"Model Run Model",
+         "description":"OasisPlatform execution mode for running an analysis, V1 = 'Single server execution' (worker versions `1.x.x` or `2.3.x`) , V2 = 'Distributed execution' scale an analysis across multiple nodes  (Only workers `2.3.x` and above)",
+         "enum":[
+            "V1",
+            "V2"
+         ]
+      },
       "model_configurable": {
          "type":"boolean",
          "title":"Model configurable",
@@ -389,6 +398,86 @@
                   "options"
                ]
             },
+            "vulnerability_set":{
+               "title":"Vulnerability set selector",
+               "description":"The 'id' field from options is used as a file suffix, e.g. vulnerability_<id>.bin, vulnerability_<id>.csv, vulnerability_<id>.parquet",
+               "type":"object",
+               "uniqueItems":false,
+               "additionalProperties": false,
+               "properties":{
+                   "name":{
+                       "type":"string",
+                       "title":"UI Option",
+                       "description":"UI name for selection",
+                       "minLength":1
+                   },
+                   "desc":{
+                       "type":"string",
+                       "title":"Short description",
+                       "description":"UI description for selection"
+                   },
+                   "used_for":{
+                      "type":"string",
+                      "title":"Where the setting is applied",
+                      "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'",
+                      "enum":[
+                         "all",
+                         "generation",
+                         "losses"
+                      ]
+                   },
+                   "tooltip":{
+                      "type":"string",
+                      "title":"UI tooltip",
+                      "description":"Long description (optional)"
+                   },
+                   "default":{
+                      "type":"string",
+                      "title":"Default footprint set",
+                      "description":"Initial setting for vulnerability set"
+                   },
+                   "options":{
+                      "type":"array",
+                      "title":"Selection options for footprint",
+                      "description":"Array of possible vulnerability sets",
+                      "items":{
+                         "type":"object",
+                         "title":"Selection option element",
+                         "description":"Vulnerability set options",
+                         "additionalProperties":false,
+                         "properties":{
+                             "id":{
+                                 "type":"string",
+                                 "title":"Vulnerability set suffix",
+                                 "description":"String value used to select a vulnerability set",
+                                 "minLength":1
+                             },
+                             "desc":{
+                                 "type":"string",
+                                 "title":"Vulnerability set description",
+                                 "description":"UI description for selection",
+                                 "minLength":1
+                             },
+                             "tooltip":{
+                                 "type":"string",
+                                 "title":"UI tooltip",
+                                 "description":"Long description (optional)"
+                             }
+                         },
+                         "required":[
+                             "id",
+                             "desc"
+                         ]
+                    }
+                 }
+              },
+              "required":[
+                 "name",
+                 "desc",
+                 "default",
+                 "options"
+              ]
+           },
             "valid_output_perspectives":{
                "type":"array",
                "title":"Globally supported loss perspectives",
@@ -449,6 +538,52 @@
                   ]
                }
             },
+            "vulnerability_adjustments": {
+               "type": "object",
+               "title": "Vulnerability Adjustments",
+               "description": "Object containing vulnerability adjustments and replacements.",
+               "properties": {
+                   "adjustments": {
+                       "type": "object",
+                       "title": "Vulnerability Factor Adjustments",
+                       "description": "Dictionary containing vulnerability adjustments with vulnerability ID as integer keys and adjustment factors as float values.",
+                       "patternProperties": {
+                           "^[0-9]+$": {
+                               "type": "number",
+                               "minimum": 0
+                           }
+                       },
+                       "additionalProperties": false
+                   },
+                   "replace_data": {
+                       "type": "object",
+                       "title": "Vulnerability Replacements",
+                       "description": "Detailed vulnerability replacements.",
+                       "patternProperties": {
+                           "^[0-9]+$": {
+                               "type": "array",
+                               "items": {
+                                   "type": "array",
+                                   "minItems": 3,
+                                   "maxItems": 3,
+                                   "items": [
+                                       { "type": "integer" },
+                                       { "type": "integer" },
+                                       { "type": "number", "minimum": 0, "maximum": 1 }
+                                   ]
+                               }
+                           }
+                       },
+                       "additionalProperties": false
+                   },
+                   "replace_file": {
+                       "type": "string",
+                       "title": "Vulnerability Replacement File",
+                       "description": "Path to a CSV file containing vulnerability data to be replaced."
+                   }
+               },
+               "additionalProperties": false
+           },                          
             "string_parameters":{
                "title":"Single string paramters",
                "type":"array",
@@ -1186,7 +1321,7 @@
                   "type":"object",
                   "title":"Assets Details",
                   "description":"Name, version and location of data",
-                  "additionalProperties":false,
+                  "additionalProperties":true,
                   "properties":{
                      "name":{
                         "type":"string",

diff --git a/ods_tools/oed/common.py b/ods_tools/oed/common.py
@@ -148,7 +148,8 @@ def fill_empty(df, columns, value):
     if isinstance(columns, str):
         columns = [columns]
     for column in columns:
-        if df[column].dtypes.name == 'category' and value not in {None, np.nan}.union(df[column].cat.categories):
+        dtype = getattr(df[column], "dtypes", getattr(df[column], "dtype", None))
+        if dtype.name == 'category' and value not in {None, np.nan}.union(df[column].cat.categories):
             df[column] = df[column].cat.add_categories(value)
         df.loc[df[column].isin(BLANK_VALUES), column] = value