add output generation

Author: vinulw

ods_tools/combine/combine.py

@@ -4,7 +4,8 @@
 import logging
 
 from ods_tools.combine.grouping import ResultGroup, create_combine_group
-from ods_tools.combine.io import get_default_output_dir, save_summary_info
+from ods_tools.combine.io import get_default_output_dir, save_output, save_summary_info
+from ods_tools.combine.output_generation import generate_alt, generate_ept
 from ods_tools.combine.result import load_analysis_dirs
 from ods_tools.combine.sampling import do_loss_sampling, generate_group_periods, generate_gpqt
 from ods_tools.combine.common import DEFAULT_CONFIG
@@ -43,6 +44,11 @@ def combine(analysis_dirs,
             group_format_priority=['M', 'Q', 'S'],
             group_correlation=None,
             occ_dtype=None,
+            group_plt=False,
+            group_alt=False,
+            group_ept=False,
+            group_ept_oep=True,
+            group_ept_aep=True,
             output_dir=None,
             **kwargs
             ):
@@ -88,6 +94,23 @@ def combine(analysis_dirs,
     # Output generation
     logger.info("Running: Output Generation")
 
+    outputs = []
+
+    if group_plt:
+        outputs.append(('plt', gplt))
+
+    if group_alt:
+        outputs.append(('alt', generate_alt(gplt, group_number_of_periods)))
+
+    if group_ept:
+        outputs.append(('ept',
+                        generate_ept(gplt, group_number_of_periods,
+                                     oep=group_ept_oep,
+                                     aep=group_ept_aep)))
+
+    for output_name, output_df in outputs:
+        save_output(output_df, output_dir, f'{output_name}.csv')
+
     return gplt
 
 

ods_tools/combine/common.py

@@ -3,6 +3,7 @@
 import os
 
 oasis_int = np.dtype(os.environ.get('OASIS_INT', 'i4'))
+oasis_float = np.dtype(os.environ.get('OASIS_FLOAT', 'f4'))
 nb_oasis_int = nb.from_dtype(oasis_int)
 
 DEFAULT_RANDOM_SEED = 8762

ods_tools/combine/config.json

@@ -3,5 +3,8 @@
   "group_number_of_periods": 100000,
   "group_mean": true,
   "group_secondary_uncertainty": true,
-  "group_format_priority": ["M", "Q", "S"]
+  "group_format_priority": ["M", "Q", "S"],
+  "group_plt": true,
+  "group_alt": true,
+  "group_ept": true
 }

ods_tools/combine/config_schema.json

@@ -114,6 +114,18 @@
       "type": "boolean",
       "default": false
       },
+    "group_ept_oep": {
+      "title": "group_ept_oep",
+      "description": "Output grouped EPT with OEP if group_ept",
+      "type": "boolean",
+      "default": true
+    },
+    "group_ept_aep": {
+      "title": "group_ept_aep",
+      "description": "Output grouped EPT with AEP if group_ept",
+      "type": "boolean",
+      "default": true
+    },
     "group_plt": {
       "title": "group_plt",
       "description": "Output grouped PLT report",

ods_tools/combine/io.py

@@ -7,13 +7,11 @@
 import logging
 from datetime import datetime
 import pandas as pd
-import os
 
-from ods_tools.combine.common import nb_oasis_int
+from ods_tools.combine.common import nb_oasis_int, oasis_float
 
 logger = logging.getLogger(__name__)
 
-oasis_float = np.dtype(os.environ.get('OASIS_FLOAT', 'f4'))
 
 DEFAULT_OCC_DTYPE = [('event_id', 'i4'),
                      ('period_no', 'i4'),
@@ -37,11 +35,21 @@ def save_summary_info(groupset_summaryinfo, groupset_info, output_dir):
 
     for gs, g_summary_info_df in groupset_summaryinfo.items():
         summary_info_fname = f'{groupset_info[gs]['perspective_code']}_GS{gs}_summary-info.csv'
-        g_summary_info_df.to_csv(Path(output_dir) / summary_info_fname, index=False)
+        save_path = Path(output_dir) / summary_info_fname
+        g_summary_info_df.to_csv(save_path, index=False)
+        logger.info(f'Saved {summary_info_fname}: ', save_path)
 
 
+def save_output(full_df, output_dir, output_name, factor_col='groupset_id', float_format='%.6f'):
+    for i in full_df[factor_col].unique():
+        save_path = output_dir / f'{i}_{output_name}'
+        full_df.query(f"{factor_col} == {i}").to_csv(save_path, index=False,
+                                                     float_format=float_format)
+        logger.info(f'Saved {output_name}: ', save_path)
+
 # occurrence reading functions from oasislmf -> copied to avoid circular imports
 
+
 @nb.jit(nopython=True, cache=True)
 def mv_read(byte_mv, cursor, _dtype, itemsize):
     """

ods_tools/combine/output_generation.py

@@ -0,0 +1,88 @@
+import pandas as pd
+import numpy as np
+
+from ods_tools.combine.common import oasis_float
+
+dtypes_al = {
+    'groupset_id': 'i4',
+    'SummaryId': 'i4',
+    'LossType': 'i4',
+    'Mean': oasis_float,
+    'Std': oasis_float
+
+}
+
+dtypes_ep = {
+    'groupset_id': 'i4',
+    'SummaryId': 'i4',
+    'EPCalc': 'i4',
+    'EPType': 'i4',
+    'RP': oasis_float,
+    'Loss': oasis_float
+}
+
+
+def generate_alt(gplt, max_period):
+    # TODO: mean loss sampling results in inf + NaN values
+    aal_group = gplt.groupby(by=["groupset_id", "SummaryId", "LossType"], as_index=False)
+
+    records = []
+    for name, group in aal_group:
+        mean_loss = group["Loss"].sum() / max_period
+        std_loss = np.sqrt(((mean_loss - group["Loss"])**2).sum() / (max_period - 1))
+
+        record = {
+            "groupset_id": name[0],
+            "SummaryId": name[1],
+            "LossType": name[2],
+            "Mean": mean_loss,
+            "Std": std_loss
+        }
+
+        records.append(record)
+
+    return pd.DataFrame(records).astype(dtypes_al)
+
+
+def assign_exceedance_probability(df, max_period):
+    original_cols = list(df.columns)
+    df["rank"] = (df.groupby(by=["groupset_id", "SummaryId", "EPCalc"], as_index=False)["Loss"]
+                  .rank(method="first", ascending=False))
+    df["RP"] = max_period / df["rank"]
+    return df[original_cols + ["RP"]]
+
+
+def generate_ept(gplt, max_group_period, oep=True, aep=True):
+    ep_groups = (
+        gplt.rename(columns={"LossType": "EPCalc"})  # check if this is the correct type
+        .groupby(by=["groupset_id", "groupeventset_id",
+                     "EventId", "GroupPeriod", "SummaryId",
+                     "EPCalc"], as_index=False)
+    )
+    grouped_df = ep_groups["Loss"].agg("sum")
+    grouped_df = grouped_df.groupby(by=["groupset_id", "SummaryId", "GroupPeriod", "EPCalc"], as_index=False)
+
+    ep_frags = []
+    if oep:
+        oep_df = (
+            grouped_df.pipe(lambda gp: gp["Loss"].max())
+            .pipe(assign_exceedance_probability, max_period=max_group_period)
+            .pipe(lambda x: x.assign(EPType=1))  # todo check OEP TVAR EPCalc 2
+        )
+
+        ep_frags.append(oep_df)
+
+    if aep:
+        aep_df = (
+            grouped_df.pipe(lambda gp: gp["Loss"].sum())
+            .pipe(assign_exceedance_probability, max_period=max_group_period)
+            .pipe(lambda x: x.assign(EPType=3))  # todo check AEP TVAR EPCalc 4
+        )
+        ep_frags.append(aep_df)
+
+    return (
+        pd.concat(ep_frags)[["groupset_id", "SummaryId", "EPCalc", "EPType", "RP", "Loss"]]
+        .astype(dtypes_ep)
+        .sort_values(by=["groupset_id", "SummaryId", "EPType", "EPCalc", "Loss"],
+                     ascending=[True, True, True, True, False])
+    )