Add cx , varobs namelists for EPSMWS obsgroup (#246)

* add missing namelist files for EPSMWS and associated ctest

* minor update to default fields

* add cx test for epsmws requested by reviewer

* add cx test for sternamwr too

* Trigger CI

Author: brettcandy

deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc

@@ -285,15 +285,15 @@ SELECT CASE (ObsGroup)
                         VarField_tcozone,VarField_solzenith,VarField_QCinfo, &
                         VarField_clw/)
   CASE (ObsGroupEPSMWS) !setup for jedi
-    Varfields(1:10) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
+    Varfields(1:9) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
                         Varfield_solzenith,Varfield_tskin, &
                         Varfield_surface,Varfield_numchans,Varfield_channum, &
-                        Varfield_mwemiss, varfield_biaspredictors /)
+                        Varfield_mwemiss /)
   CASE (ObsGroupSternaMWR) !set up for jedi
-    Varfields(1:10) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
+    Varfields(1:9) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
                         Varfield_solzenith,Varfield_tskin, &
                         Varfield_surface,Varfield_numchans,Varfield_channum, &
-                        Varfield_mwemiss, varfield_biaspredictors /)
+                        Varfield_mwemiss /)
   CASE (ObsGroupRainAccum)
     Varfields(1:1) = (/Varfield_RainAccum/)
 END SELECT

etc/global/cx/EPSMWS.nl

@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222
+/

etc/global/varobs/EPSMWS.nl

@@ -0,0 +1,3 @@
+&VarobsControlNL
+Varfields=10,11,17,19,21,54,55,28,31,80
+/

test/CMakeLists.txt

@@ -371,6 +371,10 @@ ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_sternamwr
                 YAML     varobswriter_globalnamelist_sternamwr.yaml
                 NAMELIST ../../etc/global/varobs/SternaMWR.nl
                 DATA     varobs_globalnamelist_sternamwr.nc4)
+ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_epsmws
+                YAML     varobswriter_globalnamelist_epsmws.yaml
+                NAMELIST ../../etc/global/varobs/EPSMWS.nl
+                DATA     varobs_globalnamelist_epsmws.nc4)
 
 # Tests the UKV namelist files in the etc directory
 
@@ -650,6 +654,14 @@ ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_atovs
                 YAML     cxwriter_globalnamelist_atovs.yaml
                 NAMELIST ../../etc/global/cx/ATOVS.nl
                 DATA     cx_globalnamelist_atovs.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_epsmws
+                YAML     cxwriter_globalnamelist_epsmws.yaml
+                NAMELIST ../../etc/global/cx/EPSMWS.nl
+                DATA     cx_globalnamelist_epsmws.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_sternamwr
+                YAML     cxwriter_globalnamelist_sternamwr.yaml
+                NAMELIST ../../etc/global/cx/SternaMWR.nl
+                DATA     cx_globalnamelist_sternamwr.nc4 dummy.nc4)              
 ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_ssmis
                 YAML     cxwriter_globalnamelist_ssmis.yaml
                 NAMELIST ../../etc/global/cx/SSMIS.nl

test/generate_unittest_netcdfs.py

@@ -832,6 +832,15 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                                   'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'],
                                  ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
                                  'testinput/varobs_globalnamelist_sternamwr.nc4')
+                                 
+    # EPSMWS 
+    output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle',
+                                  'MetaData/solarZenithAngle'],
+                                 ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature', 'Emiss/emissivity',
+                                  'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature',
+                                  'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'],
+                                 ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
+                                 'testinput/varobs_globalnamelist_epsmws.nc4')                             
 
 
     # GNSS-RO
@@ -1106,7 +1115,22 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                               'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels',
                               'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
                              'testinput/cx_globalnamelist_atovs.nc4')
-
+      
+    # EPSMWS                       
+    output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m',
+                              'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'],
+                             ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water',
+                              'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels',
+                              'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
+                             'testinput/cx_globalnamelist_epsmws.nc4')
+    # SternaMWR                      
+    output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m',
+                              'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'],
+                             ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water',
+                              'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels',
+                              'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
+                             'testinput/cx_globalnamelist_sternamwr.nc4')
+                             
     # SSMIS
     output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m',
                               'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'],

test/testinput/cx_globalnamelist_epsmws.nc4

@@ -0,0 +1,42 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: EPSMWS
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/cx_globalnamelist_epsmws.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: ../etc/global/cx
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1","2","3","4","5","6","13","16","17"]
+      expected_upper_air_variables: ["1","5","11","29","30","31","34","35"]
+      expected_main_table_columns:
+      - # batch 1 - 9 1dvalues first ; then 2d values in expected order
+        # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns
+        - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"]  # column 1 - 1st observation
+        - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"]  # column 2 - 2nd observation
+        - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column 3 - observation 4
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0

test/testinput/cx_globalnamelist_sternamwr.nc4

@@ -0,0 +1,42 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: SternaMWR
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/cx_globalnamelist_sternamwr.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: ../etc/global/cx
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1","2","3","4","5","6","13","16","17"]
+      expected_upper_air_variables: ["1","5","11","29","30","31","34","35"]
+      expected_main_table_columns:
+      - # batch 1 - 9 1dvalues first ; then 2d values in expected order
+        # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns
+        - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"]  # column 1 - 1st observation
+        - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"]  # column 2 - 2nd observation
+        - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column 3 - observation 4
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0