Several fixes and adds some logging about non compliant files

Signed-off-by: Alexis Jeandet <[email protected]>

Author: jeandet

pyistp/_impl.py

@@ -4,30 +4,44 @@
 import re
 import numpy as np
 from typing import List
+import logging
 
 DEPEND_REGEX = re.compile("DEPEND_\\d")
 
+ISTP_NOT_COMPLIANT_W = "Non compliant ISTP file"
+
+log = logging.getLogger(__name__)
+
 
 def _get_attributes(cdf: object, var: str):
     attrs = {}
     for attr in cdf.variable_attributes(var):
+        value = cdf.variable_attribute_value(var, attr)
         if attr.endswith("_PTR") or attr[:-1].endswith("_PTR_"):
-            value = cdf.values(cdf.variable_attribute_value(var, attr))
-            if hasattr(value, 'tolist'):
-                attrs[attr] = value.tolist()
+            if cdf.has_variable(value):
+                value = cdf.values(value)
+                if hasattr(value, 'tolist'):
+                    attrs[attr] = value.tolist()
+                else:
+                    attrs[attr] = value
             else:
+                log.warning(
+                    f"{ISTP_NOT_COMPLIANT_W}: variable {var} has {attr} attribute which points to a variable which does not exist")
                 attrs[attr] = value
         else:
-            attrs[attr] = cdf.variable_attribute_value(var, attr)
+            attrs[attr] = value
     return attrs
 
 
 def _get_axis(cdf: object, var: str):
-    if cdf.is_char(var):
-        if 'sig_digits' in cdf.variable_attributes(var):  # cluster CSA trick :/
-            return SupportDataVariable(name=var, values=np.asarray(cdf.values(var), dtype=float),
-                                       attributes=_get_attributes(cdf, var))
-    return SupportDataVariable(name=var, values=cdf.values(var), attributes=_get_attributes(cdf, var))
+    if cdf.has_variable(var):
+        if cdf.is_char(var):
+            if 'sig_digits' in cdf.variable_attributes(var):  # cluster CSA trick :/
+                return SupportDataVariable(name=var, values=np.asarray(cdf.values(var), dtype=float),
+                                           attributes=_get_attributes(cdf, var))
+        return SupportDataVariable(name=var, values=cdf.values(var), attributes=_get_attributes(cdf, var))
+    log.warning(f"{ISTP_NOT_COMPLIANT_W}: trying to load {var} as support data but it is absent from the file")
+    return None
 
 
 def _get_axes(cdf: object, var: str, data_shape):
@@ -39,8 +53,8 @@ def _get_axes(cdf: object, var: str, data_shape):
         if len(unix_time) == data_shape[0]:
             unix_time.values = (unix_time.values * 1e9).astype('<M8[ns]')
             axes[0] = unix_time
-            Warning(
-                "Non compliant CDF file, swapping DEPEND_0 with DEPEND_TIME, if you think this is a bug report it here: https://github.com/SciQLop/PyISTP/issues")
+            log.warning(
+                f"{ISTP_NOT_COMPLIANT_W}: swapping DEPEND_0 with DEPEND_TIME, if you think this is a bug report it here: https://github.com/SciQLop/PyISTP/issues")
     return axes
 
 
@@ -56,6 +70,9 @@ def _load_data_var(cdf: object, var: str) -> DataVariable or None:
     axes = _get_axes(cdf, var, values.shape)
     attributes = _get_attributes(cdf, var)
     labels = _get_labels(attributes)
+    if len(axes) == 0:
+        log.warning(f"{ISTP_NOT_COMPLIANT_W}: {var} was marked as data variable but it has 0 support variable")
+        return None
     if None in axes or axes[0].values.shape[0] != values.shape[0]:
         return None
     return DataVariable(name=var, values=values, attributes=attributes, axes=axes, labels=labels)

pyistp/drivers/pycdfpp.py

@@ -20,7 +20,10 @@ def variables(self):
             return list(self.cdf)
         return []
 
-    def is_char(self,var):
+    def has_variable(self, name):
+        return name in self.cdf
+
+    def is_char(self, var):
         return self.cdf[var].type == pycdfpp.CDF_CHAR
 
     def variable_attributes(self, var):

pyistp/drivers/spacepy.py

@@ -28,6 +28,9 @@ def variables(self):
             return self.cdf.keys()
         return []
 
+    def has_variable(self, name):
+        return name in self.cdf
+
     def variable_attributes(self, var):
         if self.cdf:
             return self.cdf[var].attrs.keys()

pyistp/support_data_variable.py

@@ -1,9 +1,14 @@
+import numpy as np
+
+
 class SupportDataVariable:
     __slots__ = ("name", "values", 'attributes')
 
     def __init__(self, name, values, attributes):
         self.name = name
         self.values = values
+        if type(self.values) is list:
+            self.values = np.array(self.values)
         self.attributes = attributes
 
     def __len__(self):

tests/resources/c3_cp_efw_l3_e3d_inert_00000000_v01.cdf

@@ -58,7 +58,11 @@
       "tha_peer_vthermal", "tha_peer_ptens", "tha_peeb_magfQ"]),
     (f"{current_path}/resources/thd_l2_efi_00000000_v01.cdf",
      ['thd_eff_e34_efs', 'thd_eff_q_mag', 'thd_efs_dot0_gsm', 'thd_efs_dot0_dsl', 'thd_efs_dot0_gse', 'thd_eff_q_pha',
-      'thd_eff_dot0_gse', 'thd_eff_e12_efs', 'thd_eff_dot0_dsl', 'thd_eff_dot0_gsm'])
+      'thd_eff_dot0_gse', 'thd_eff_e12_efs', 'thd_eff_dot0_dsl', 'thd_eff_dot0_gsm']),
+    (f"{current_path}/resources/c3_cp_efw_l3_e3d_inert_00000000_v01.cdf",
+     ['delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT', 'E_Vec_xyz_ISR2__C3_CP_EFW_L3_E3D_INERT',
+      'E_sigma__C3_CP_EFW_L3_E3D_INERT']
+     )
 )