From b10415edd6d7b2e6c45a3c8b253f8ecc83df06c3 Mon Sep 17 00:00:00 2001 From: morobking Date: Tue, 25 Nov 2025 17:49:06 +0000 Subject: [PATCH 1/4] coordinate issue fixed --- .../data/cmor_config/ESACCI-SOILMOISTURE.yml | 19 ++++++++++++++----- .../datasets/esacci_soilmoisture.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml index 0b17a06e94..562a8e5186 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml @@ -10,11 +10,20 @@ attributes: version: 'L3S-SSMV-COMBINED-v08.1' comment: '' variables: +# sm: +# mip: Eday +# raw: sm +# filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-{year}????000000-fv08.1.nc +# smStderr: +# mip: Eday +# raw: sm_uncertainty +# filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-{year}????000000-fv08.1.nc + sm: mip: Eday raw: sm - filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-{year}????000000-fv08.1.nc - smStderr: - mip: Eday - raw: sm_uncertainty - filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-{year}????000000-fv08.1.nc + filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc + # smStderr: + # mip: Eday + # raw: sm_uncertainty + # filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc \ No newline at end of file diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py index 0676c5c85f..74e591fb35 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py @@ -57,6 +57,16 @@ def fix_coords(cube): cube: iris.cube.Cube data cube with fixed coordinates. """ + # This makes fix_dim_coordnames work with GAPFILLED + try: + cube.coord('lat').standard_name = 'latitude' + except: + pass + try: + cube.coord('lon').standard_name = 'longitude' + except: + pass + # First fix any completely missing coord var names fix_dim_coordnames(cube) From 35e9c88cc597b48e1dfe804f2d4b92fe8d12b0f8 Mon Sep 17 00:00:00 2001 From: morobking Date: Tue, 9 Dec 2025 18:37:38 +0000 Subject: [PATCH 2/4] Working version for gapfilled SM. need to tidy ready for PR --- .../data/cmor_config/ESACCI-SOILMOISTURE.yml | 20 ++++++++++++++----- .../datasets/esacci_soilmoisture.py | 15 ++++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml index 562a8e5186..3dc5a55041 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml @@ -5,10 +5,18 @@ attributes: modeling_realm: sat institution: 'TU Wien (AUT); VanderSat B.V. (NL); Planet Labs (NL); CESBIO (FR), EODC Gmbh (AUT)' reference: 'esacci-soilmoisture' + comment: '' + + # COMBINED Attributes source: 'ftp://anon-ftp.ceda.ac.uk/neodc/esacci/soil_moisture/data/' title: 'ESA CCI Soil Moisture' version: 'L3S-SSMV-COMBINED-v08.1' - comment: '' + + # GAPFILLED Attributes + version 'L3S-SSMV-COMBINED_GAPFILLED-fv09.1' + source: 'https://researchdata.tuwien.ac.at/records/s5j4q-rpd32' + title: 'ESA CCI Soil Moisture' + variables: # sm: # mip: Eday @@ -22,8 +30,10 @@ variables: sm: mip: Eday raw: sm + gapfilled: True filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc - # smStderr: - # mip: Eday - # raw: sm_uncertainty - # filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc \ No newline at end of file + smStderr: + mip: Eday + raw: sm_uncertainty + gapfilled: True + filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc \ No newline at end of file diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py index 74e591fb35..6c0cf0bf53 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py @@ -96,14 +96,18 @@ def fix_coords(cube): return cube -def extract_variable(raw_info): - """Extract variables.""" +def extract_variable(raw_info, is_gapfilled): + """Extract variables. + is_gapfilled is True or False to determine if the StdErr is going to + load as an ancillary variable or a seperate cube. + """ rawvar = raw_info["name"] constraint = iris.Constraint(name=rawvar) - if rawvar == "sm_uncertainty": + if rawvar == "sm_uncertainty" and is_gapfilled == False: sm_cube = iris.load_cube( raw_info["file"], iris.NameConstraint(var_name="sm") ) + ancillary_var = sm_cube.ancillary_variable( "Volumetric Soil Moisture Uncertainty" ) @@ -112,6 +116,7 @@ def extract_variable(raw_info): cube = iris.load_cube(raw_info["file"], constraint) # Remove dysfunctional ancillary data without standard names + for ancillary_variable in cube.ancillary_variables(): cube.remove_ancillary_variable(ancillary_variable) @@ -136,17 +141,19 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): var_info = cfg["cmor_table"].get_variable(vals["mip"], var_name) glob_attrs["mip"] = vals["mip"] raw_info = {"name": vals["raw"]} + is_gapfilled = vals["gapfilled"] inpfile_pattern = os.path.join(in_dir, vals["filename"]) logger.info( "CMORizing var %s from file type %s", var_name, inpfile_pattern ) + logger.info(f"{cfg=}") for year in range(start_date.year, end_date.year + 1): year_inpfile_pattern = inpfile_pattern.format(year=year) inpfiles = sorted(glob.glob(year_inpfile_pattern)) for inpfile in inpfiles: raw_info["file"] = inpfile - cube = extract_variable(raw_info) + cube = extract_variable(raw_info, is_gapfilled) all_data_cubes.append(cube) final_cube = concatenate(all_data_cubes) fix_var_metadata(final_cube, var_info) From 1fba34c714612439d21d678294901697006fea4d Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 10 Dec 2025 12:29:08 +0000 Subject: [PATCH 3/4] update after style check --- .../data/formatters/datasets/esacci_soilmoisture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py index 6c0cf0bf53..cb5b51a6a3 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py @@ -60,11 +60,11 @@ def fix_coords(cube): # This makes fix_dim_coordnames work with GAPFILLED try: cube.coord('lat').standard_name = 'latitude' - except: + except iris.exceptions.CoordinateNotFoundError: pass try: cube.coord('lon').standard_name = 'longitude' - except: + except iris.exceptions.CoordinateNotFoundError: pass # First fix any completely missing coord var names @@ -103,7 +103,7 @@ def extract_variable(raw_info, is_gapfilled): """ rawvar = raw_info["name"] constraint = iris.Constraint(name=rawvar) - if rawvar == "sm_uncertainty" and is_gapfilled == False: + if rawvar == "sm_uncertainty" and not is_gapfilled: sm_cube = iris.load_cube( raw_info["file"], iris.NameConstraint(var_name="sm") ) @@ -146,7 +146,6 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): logger.info( "CMORizing var %s from file type %s", var_name, inpfile_pattern ) - logger.info(f"{cfg=}") for year in range(start_date.year, end_date.year + 1): year_inpfile_pattern = inpfile_pattern.format(year=year) @@ -179,4 +178,4 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): out_dir, glob_attrs, unlimited_dimensions=["time"], - ) + ) \ No newline at end of file From 93ff8c3428c2a751ad1572d42aba028be05bee32 Mon Sep 17 00:00:00 2001 From: morobking Date: Wed, 10 Dec 2025 15:22:40 +0000 Subject: [PATCH 4/4] pre commit run --- .../data/cmor_config/ESACCI-SOILMOISTURE.yml | 6 +++--- .../data/formatters/datasets/esacci_soilmoisture.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml index 3dc5a55041..651fed0932 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-SOILMOISTURE.yml @@ -6,7 +6,7 @@ attributes: institution: 'TU Wien (AUT); VanderSat B.V. (NL); Planet Labs (NL); CESBIO (FR), EODC Gmbh (AUT)' reference: 'esacci-soilmoisture' comment: '' - + # COMBINED Attributes source: 'ftp://anon-ftp.ceda.ac.uk/neodc/esacci/soil_moisture/data/' title: 'ESA CCI Soil Moisture' @@ -16,7 +16,7 @@ attributes: version 'L3S-SSMV-COMBINED_GAPFILLED-fv09.1' source: 'https://researchdata.tuwien.ac.at/records/s5j4q-rpd32' title: 'ESA CCI Soil Moisture' - + variables: # sm: # mip: Eday @@ -36,4 +36,4 @@ variables: mip: Eday raw: sm_uncertainty gapfilled: True - filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc \ No newline at end of file + filename: ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED_GAPFILLED-{year}????000000-fv09.1.nc diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py index cb5b51a6a3..abec237523 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_soilmoisture.py @@ -59,14 +59,14 @@ def fix_coords(cube): """ # This makes fix_dim_coordnames work with GAPFILLED try: - cube.coord('lat').standard_name = 'latitude' + cube.coord("lat").standard_name = "latitude" except iris.exceptions.CoordinateNotFoundError: pass try: - cube.coord('lon').standard_name = 'longitude' + cube.coord("lon").standard_name = "longitude" except iris.exceptions.CoordinateNotFoundError: pass - + # First fix any completely missing coord var names fix_dim_coordnames(cube) @@ -107,7 +107,7 @@ def extract_variable(raw_info, is_gapfilled): sm_cube = iris.load_cube( raw_info["file"], iris.NameConstraint(var_name="sm") ) - + ancillary_var = sm_cube.ancillary_variable( "Volumetric Soil Moisture Uncertainty" ) @@ -116,7 +116,7 @@ def extract_variable(raw_info, is_gapfilled): cube = iris.load_cube(raw_info["file"], constraint) # Remove dysfunctional ancillary data without standard names - + for ancillary_variable in cube.ancillary_variables(): cube.remove_ancillary_variable(ancillary_variable) @@ -178,4 +178,4 @@ def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): out_dir, glob_attrs, unlimited_dimensions=["time"], - ) \ No newline at end of file + )