Oasislmf ci update

.github/workflows/model-tests.yml

@@ -10,10 +10,18 @@ on:
       oasislmf_branch:
         description: 'oasislmf git branch to install (leave empty for latest PyPI release)'
         required: false
-        default: ''
+        default: 'main'
       ods_branch:
         description: 'ODS_Tools git branch to install (leave empty for version bundled with oasislmf)'
         required: false
+        default: 'main'
+      oasis_data_manager_branch:
+        description: 'oasis-data-manager git branch to install (leave empty to skip)'
+        required: false
+        default: 'develop'
+      pytest_params:
+        description: 'Extra pytest parameters (e.g. -k testname)'
+        required: false
         default: ''
 
 jobs:
@@ -30,32 +38,30 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install oasislmf (PyPI)
-        if: ${{ !inputs.oasislmf_branch }}
-        run: pip install oasislmf[extra]
-
       - name: Install oasislmf (git branch)
-        if: ${{ inputs.oasislmf_branch }}
-        run: pip install "oasislmf[extra] @ git+https://github.com/OasisLMF/OasisLMF.git@${{ inputs.oasislmf_branch }}"
+        run: pip install "oasislmf[extra] @ git+https://github.com/OasisLMF/OasisLMF.git@${{ inputs.ods_branch || 'main' }}"
 
       - name: Install ODS_Tools (git branch)
-        if: ${{ inputs.ods_branch }}
-        run: pip install "git+https://github.com/OasisLMF/ODS_Tools.git@${{ inputs.ods_branch }}"
+        #if: ${{ inputs.ods_branch }}
+        run: pip uninstall ods-tools -y && pip install "git+https://github.com/OasisLMF/ODS_Tools.git@${{ inputs.ods_branch || 'main' }}"
+
+      - name: Install oasis-data-manager (git branch)
+        #if: ${{ inputs.oasis_data_manager_branch }}
+        run: pip uninstall oasis-data-manager -y && pip install "git+https://github.com/OasisLMF/OasisDataManager.git@${{ inputs.oasis_data_manager_branch  || 'develop' }}"
 
       - name: Install test dependencies
-        run: pip install pytest
+        run: pip install pytest s3fs
 
       - name: Install Complex Model wrapper
-        run: cd PiWindComplexModel && pip install . 
+        run: cd PiWindComplexModel && pip install .
 
       - name: Run model tests
-        run: pytest tests/test_model_runs.py -v --check-results --basetemp=pytest-tmp
+        run: pytest tests/test_model_runs.py -v --check-results --basetemp=pytest-tmp ${{ inputs.pytest_params }}
 
       - name: Archive model logs
         if: always()
         run: |
-          find pytest-tmp -path "*/run/log" -type d \
-            | tar -czf model-logs.tar.gz --files-from=-
+          find pytest-tmp -path "*/run/log" -o -path "*/run/output" -type d | tar -czf model-logs.tar.gz --files-from=-
 
       - name: Upload model logs
         if: always()

PiWindS3/keys_data/lookup_config.json

@@ -6,10 +6,11 @@
     },
     "builtin_lookup_type": "new_lookup",
     "keys_data_storage": {
-        "storage_class": "oasis_data_manager.filestore.backends.aws_s3.AwsS3Storage",
+        "storage_class": "oasis_data_manager.filestore.backends.aws.AwsS3Storage",
         "options":{
-            "bucket_name": "oasislmf-model-library-oasis-piwind/keys_data",
-            "region_name": "eu-west-1"
+            "bucket_name": "oasislmf-model-library-oasis-piwind-ci-testing/keys_data",
+            "region_name": "eu-west-1",
+            "public_bucket": true
         }
     },
     "step_definition": {
@@ -43,6 +44,13 @@
             "parameters": {"file_path": "vulnerability_dict.csv",
                            "id_columns": ["vulnerability_id"]
                           }
+        },
+            "amplification": {
+            "type": "merge",
+            "columns": ["peril_id", "postalcode"],
+            "parameters": {"file_path": "%%KEYS_DATA_PATH%%/amplification_dict.csv",
+                           "id_columns": ["amplification_id"]
+                          }
         }
     },
     "strategy": ["split_loc_perils_covered", "peril", "create_coverage_type", "vulnerability"]

PiWindS3/meta_data/model_settings.json

@@ -1,4 +1,5 @@
 {
+    "version": "3",
     "model_settings":{
         "event_set":{
             "name": "Event Set",
@@ -19,14 +20,19 @@
      },
     "lookup_settings":{
         "supported_perils":[
-           {"id": "WSS", "desc": "Single Peril: Storm Surge"},
-           {"id": "WTC", "desc": "Single Peril: Tropical Cyclone"},
+           {"id": "WSS", "desc": "Single Peril: Storm Surge", "peril_correlation_group": 1},
+           {"id": "WTC", "desc": "Single Peril: Tropical Cyclone", "peril_correlation_group": 2},
            {"id": "WW1", "desc": "Group Peril: Windstorm with storm surge"},
            {"id": "WW2", "desc": "Group Peril: Windstorm w/o storm surge"}
         ]
     },
-    "data_settings": {
-	"damage_group_fields": ["PortNumber", "AccNumber", "LocNumber"]
-    },
+    "correlation_settings": [
+          {"peril_correlation_group":  1, "damage_correlation_value":  "0.7", "hazard_correlation_value":  "0.0"},
+          {"peril_correlation_group":  2, "damage_correlation_value":  "0.5", "hazard_correlation_value":  "0.0"}
+      ],
+      "data_settings": {
+          "damage_group_fields": ["PortNumber", "AccNumber", "LocNumber"],
+          "hazard_group_fields": ["PortNumber", "AccNumber", "LocNumber"]
+      },
     "model_default_samples" :10
 }

PiWindS3/model_data/model_storage.json

@@ -1,7 +1,8 @@
 {
-    "storage_class": "oasis_data_manager.filestore.backends.aws_s3.AwsS3Storage",
+    "storage_class": "oasis_data_manager.filestore.backends.aws.AwsS3Storage",
     "options":{
-        "bucket_name": "oasislmf-model-library-oasis-piwind/model_data",
-        "region_name": "eu-west-1"
+        "bucket_name": "oasislmf-model-library-oasis-piwind-ci-testing/model_data",
+        "region_name": "eu-west-1",
+        "public_bucket": true
     }
 }

PiWindS3/tests/test_1/analysis_settings.json

@@ -35,17 +35,5 @@
             }
         }
     ],
-    "ri_output": true,
-    "ri_summaries": [
-        {
-            "id": 1,
-            "ord_output": {
-                "alt_period": true,
-                "elt_sample": true,
-                "ept_full_uncertainty_aep": true,
-                "ept_full_uncertainty_oep": true,
-                "parquet_format": false
-            }
-        }
-    ]
+    "ri_output": false
 }

PiWindS3/tests/test_1/expected_results/output/analysis_settings.json

@@ -0,0 +1,90 @@
+{
+    "analysis_tag": "base_example",
+    "source_tag": "MDK",
+    "model_name_id": "PiWind",
+    "model_supplier_id": "OasisLMF",
+    "gul_threshold": 0,
+    "number_of_samples": 10,
+    "gul_output": true,
+    "model_settings": {
+        "event_set": "p",
+        "event_occurrence_id": "lt"
+    },
+    "gul_summaries": [
+        {
+            "id": 1,
+            "ord_output": {
+                "alt_period": true,
+                "elt_sample": true,
+                "ept_full_uncertainty_aep": true,
+                "ept_full_uncertainty_oep": true,
+                "parquet_format": false
+            }
+        }
+    ],
+    "il_output": true,
+    "il_summaries": [
+        {
+            "id": 1,
+            "ord_output": {
+                "alt_period": true,
+                "elt_sample": true,
+                "ept_full_uncertainty_aep": true,
+                "ept_full_uncertainty_oep": true,
+                "parquet_format": false
+            }
+        }
+    ],
+    "ri_output": false,
+    "version": "3",
+    "lookup_settings": {
+        "supported_perils": [
+            {
+                "id": "WSS",
+                "desc": "Single Peril: Storm Surge",
+                "peril_correlation_group": 1
+            },
+            {
+                "id": "WTC",
+                "desc": "Single Peril: Tropical Cyclone",
+                "peril_correlation_group": 2
+            },
+            {
+                "id": "WW1",
+                "desc": "Group Peril: Windstorm with storm surge"
+            },
+            {
+                "id": "WW2",
+                "desc": "Group Peril: Windstorm w/o storm surge"
+            }
+        ]
+    },
+    "correlation_settings": [
+        {
+            "peril_correlation_group": 1,
+            "damage_correlation_value": "0.7",
+            "hazard_correlation_value": "0.0"
+        },
+        {
+            "peril_correlation_group": 2,
+            "damage_correlation_value": "0.5",
+            "hazard_correlation_value": "0.0"
+        }
+    ],
+    "data_settings": {
+        "damage_group_fields": [
+            "PortNumber",
+            "AccNumber",
+            "LocNumber"
+        ],
+        "hazard_group_fields": [
+            "PortNumber",
+            "AccNumber",
+            "LocNumber"
+        ]
+    },
+    "model_default_samples": 10,
+    "ri_summaries": [],
+    "rl_output": false,
+    "rl_summaries": []
+}