From 7d69f3637e7fb3d441666243a33633f008ec48cb Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Tue, 16 Dec 2025 23:10:46 +0000 Subject: [PATCH 1/7] #282 Enabled switching of CDDS extract and expanded unit tests --- CMEW/app/configure_for/rose-app.conf | 2 +- .../bin/create_request_file.py | 12 ++++ .../bin/test_create_request_file.py | 57 ++++++++++++++++++- CMEW/app/configure_standardise/rose-app.conf | 2 +- CMEW/app/housekeeping/rose-app.conf | 2 +- CMEW/app/standardise_model_data/rose-app.conf | 2 +- CMEW/app/unittest/rose-app.conf | 2 +- CMEW/flow.cylc | 12 +++- CMEW/rose-suite.conf | 2 + 9 files changed, 84 insertions(+), 9 deletions(-) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index efcd48a4..56705c8f 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2024-2025, Met Office. +# (C) Crown Copyright 2022-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index 2e33b739..c06287af 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -17,9 +17,16 @@ def create_request(): configparser.ConfigParser() CDDS request configuration. """ + extract_flag = os.environ.get("EXTRACT", "true").lower() == "true" + extract_data_path = os.environ.get("EXTRACT_DATA_PATH", "").strip() end_year = int(os.environ["START_YEAR"]) + int( os.environ["NUMBER_OF_YEARS"] ) + if not extract_flag and not extract_data_path: + raise ValueError( + "EXTRACT=False but EXTRACT_DATA_PATH is empty. " + "Provide a full path to previously extracted model output." + ) request = configparser.ConfigParser() request["metadata"] = { "base_date": "1850-01-01T00:00:00", @@ -47,6 +54,8 @@ def create_request(): "root_data_dir": os.environ["ROOT_DATA_DIR"], "workflow_basename": "CMEW", } + if not extract_flag: + request["common"]["root_data_dir"] = extract_data_path request["data"] = { "end_date": f"{end_year}-01-01T00:00:00", "mass_data_class": "crum", @@ -65,6 +74,9 @@ def create_request(): "skip_archive": "True", "cylc_args": "--no-detach -v", } + if not extract_flag: + request["conversion"]["skip_extract"] = "True" + return request diff --git a/CMEW/app/configure_standardise/bin/test_create_request_file.py b/CMEW/app/configure_standardise/bin/test_create_request_file.py index 5ac148c0..63251773 100644 --- a/CMEW/app/configure_standardise/bin/test_create_request_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_request_file.py @@ -1,12 +1,16 @@ # (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. + import os +import pytest from create_request_file import create_request - -def test_create_request(monkeypatch): - # In the order defined in 'create_request_file.py'. +def _set_base_env(monkeypatch): + """ + Set the base environment variables needed by create_request(). + In the order defined in 'create_request_file.py'. + """ monkeypatch.setenv("START_YEAR", "1993") monkeypatch.setenv("NUMBER_OF_YEARS", "1") monkeypatch.setenv("CALENDAR", "360_day") @@ -17,10 +21,22 @@ def test_create_request(monkeypatch): monkeypatch.setenv("SUITE_ID", "u-az513") monkeypatch.setenv("VARIABLES_PATH", "/path/to/variables.txt") +def _clear_extract_env(monkeypatch): + """Ensure EXTRACT/EXTRACT_DATA_PATH do not leak from the suite env.""" + monkeypatch.delenv("EXTRACT", raising=False) + monkeypatch.delenv("EXTRACT_DATA_PATH", raising=False) + +def test_create_request_default_extract(monkeypatch): + """EXTRACT default (True) - no skip_extract, root_data_dir unchanged.""" + _clear_extract_env(monkeypatch) + _set_base_env(monkeypatch) + # Do not set EXTRACT or EXTRACT_DATA_PATH → EXTRACT defaults to True. + config = create_request() actual = { section: dict(config.items(section)) for section in config.sections() } + expected = { "metadata": { "branch_method": "no parent", @@ -65,6 +81,41 @@ def test_create_request(monkeypatch): "mip_convert_plugin": "UKESM1", "skip_archive": "True", "cylc_args": "--no-detach -v", + # NOTE: no 'skip_extract' key when EXTRACT defaults to True }, } + assert actual == expected + +def test_create_request_extract_false_with_path(monkeypatch): + """EXTRACT=False + EXTRACT_DATA_PATH set → skip_extract + override root_data_dir.""" + _clear_extract_env(monkeypatch) + _set_base_env(monkeypatch) + monkeypatch.setenv("EXTRACT", "false") + monkeypatch.setenv("EXTRACT_DATA_PATH", "/pre/extracted/data") + + config = create_request() + actual = { + section: dict(config.items(section)) for section in config.sections() + } + + # Ensure skip_extract is set in [conversion] + assert actual["conversion"]["skip_extract"] == "True" + + # Ensure root_data_dir is taken from EXTRACT_DATA_PATH, not ROOT_DATA_DIR + assert actual["common"]["root_data_dir"] == "/pre/extracted/data" + + # Optional extra checks to ensure other keys unaffected: + assert actual["common"]["root_proc_dir"] == "/path/to/proc/dir/" + assert actual["conversion"]["mip_convert_plugin"] == "UKESM1" + +def test_create_request_extract_false_without_path_raises(monkeypatch): + """EXTRACT=False with no EXTRACT_DATA_PATH → fail with ValueError.""" + _clear_extract_env(monkeypatch) + _set_base_env(monkeypatch) + monkeypatch.setenv("EXTRACT", "false") + # EXTRACT_DATA_PATH intentionally not set (or could set to empty/whitespace) + + with pytest.raises(ValueError, match="EXTRACT=False"): + create_request() + diff --git a/CMEW/app/configure_standardise/rose-app.conf b/CMEW/app/configure_standardise/rose-app.conf index ecd720ef..a794e645 100644 --- a/CMEW/app/configure_standardise/rose-app.conf +++ b/CMEW/app/configure_standardise/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2024-2025, Met Office. +# (C) Crown Copyright 2022-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/housekeeping/rose-app.conf b/CMEW/app/housekeeping/rose-app.conf index 6b2bb74c..8e21fed5 100644 --- a/CMEW/app/housekeeping/rose-app.conf +++ b/CMEW/app/housekeeping/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2024-2025, Met Office. +# (C) Crown Copyright 2022-2025, Met Office. # The LICENSE.md file contains full licensing details. meta=rose_prune diff --git a/CMEW/app/standardise_model_data/rose-app.conf b/CMEW/app/standardise_model_data/rose-app.conf index 12fe8324..154d4830 100644 --- a/CMEW/app/standardise_model_data/rose-app.conf +++ b/CMEW/app/standardise_model_data/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2024-2025, Met Office. +# (C) Crown Copyright 2022-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/unittest/rose-app.conf b/CMEW/app/unittest/rose-app.conf index e81f016b..591a560b 100644 --- a/CMEW/app/unittest/rose-app.conf +++ b/CMEW/app/unittest/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2024-2025, Met Office. +# (C) Crown Copyright 2022-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index f95aa765..79567cff 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -21,7 +21,8 @@ install_env_file => configure_recipe & configure_for configure_for => configure_standardise configure_recipe => run_recipe - configure_standardise => standardise_model_data => run_recipe & housekeeping + configure_standardise => standardise_model_data + standardise_model_data => run_recipe & housekeeping {%- if TEST %} run_recipe => compare {%- endif %} @@ -35,6 +36,8 @@ platform = localhost [[[environment]]] ESMVALTOOL_MODULE_NAME = {{ ESMVALTOOL_MODULE_NAME }} + EXTRACT = {{ EXTRACT }} + EXTRACT_DATA_PATH = {{ EXTRACT_DATA_PATH }} USER_CONFIG_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/etc USER_CONFIG_PATH = ${USER_CONFIG_DIR}/config-user.yml OUTPUT_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/cycle/${CYLC_TASK_CYCLE_POINT} @@ -56,6 +59,13 @@ # Workaround for bug in CDDS: ROOT_SOFTWARE_DIR: unbound variable. ROOT_SOFTWARE_DIR = ${CDDS_SOFTWARE_DIR} CDDS_VERSION = {{ CDDS_VERSION }} + {%- if EXTRACT in (False, "false", "False") %} + # When EXTRACT is False, reuse pre-extracted data from EXTRACT_DATA_PATH. + # Align ROOT_DATA_DIR (used by restructure_for_cmip6) with the path + # that create_request_file.py writes into [common].root_data_dir. + ROOT_DATA_DIR = "{{ EXTRACT_DATA_PATH }}" + {%- endif %} + [[install_env_file]] [[[environment]]] diff --git a/CMEW/rose-suite.conf b/CMEW/rose-suite.conf index 83fd302d..7def6bfc 100644 --- a/CMEW/rose-suite.conf +++ b/CMEW/rose-suite.conf @@ -14,6 +14,8 @@ DRS_OBS="default" DRS_OBS4MIPS="default" DRS_OBS6="default" ESMVALTOOL_MODULE_NAME="" +EXTRACT=true +EXTRACT_DATA_PATH="" MAX_PARALLEL_TASKS=4 MODEL_ID="UKESM1-0-LL" NUMBER_OF_YEARS=1 From c02290e702e71de9791adbfc77456926bf8191b9 Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Tue, 16 Dec 2025 23:43:36 +0000 Subject: [PATCH 2/7] Fix pre-commit issues in test_create_request_file.py --- .../configure_standardise/bin/test_create_request_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMEW/app/configure_standardise/bin/test_create_request_file.py b/CMEW/app/configure_standardise/bin/test_create_request_file.py index 63251773..4eb22569 100644 --- a/CMEW/app/configure_standardise/bin/test_create_request_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_request_file.py @@ -88,7 +88,8 @@ def test_create_request_default_extract(monkeypatch): assert actual == expected def test_create_request_extract_false_with_path(monkeypatch): - """EXTRACT=False + EXTRACT_DATA_PATH set → skip_extract + override root_data_dir.""" + """EXTRACT=False + EXTRACT_DATA_PATH set → + skip_extract + override root_data_dir.""" _clear_extract_env(monkeypatch) _set_base_env(monkeypatch) monkeypatch.setenv("EXTRACT", "false") @@ -114,7 +115,7 @@ def test_create_request_extract_false_without_path_raises(monkeypatch): _clear_extract_env(monkeypatch) _set_base_env(monkeypatch) monkeypatch.setenv("EXTRACT", "false") - # EXTRACT_DATA_PATH intentionally not set (or could set to empty/whitespace) + # EXTRACT_DATA_PATH intentionally not set (or could set to empty) with pytest.raises(ValueError, match="EXTRACT=False"): create_request() From 011ce984d86dde91a67f1f55fdc1d9fafb263cb9 Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Wed, 17 Dec 2025 00:01:14 +0000 Subject: [PATCH 3/7] Fixed reformatting --- .../configure_standardise/bin/create_request_file.py | 4 ++-- .../bin/test_create_request_file.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index c06287af..19506bb2 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -24,8 +24,8 @@ def create_request(): ) if not extract_flag and not extract_data_path: raise ValueError( - "EXTRACT=False but EXTRACT_DATA_PATH is empty. " - "Provide a full path to previously extracted model output." + "EXTRACT=False but EXTRACT_DATA_PATH is empty. " + "Provide a full path to previously extracted model output." ) request = configparser.ConfigParser() request["metadata"] = { diff --git a/CMEW/app/configure_standardise/bin/test_create_request_file.py b/CMEW/app/configure_standardise/bin/test_create_request_file.py index 4eb22569..e2d5a18e 100644 --- a/CMEW/app/configure_standardise/bin/test_create_request_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_request_file.py @@ -6,6 +6,7 @@ from create_request_file import create_request + def _set_base_env(monkeypatch): """ Set the base environment variables needed by create_request(). @@ -21,11 +22,13 @@ def _set_base_env(monkeypatch): monkeypatch.setenv("SUITE_ID", "u-az513") monkeypatch.setenv("VARIABLES_PATH", "/path/to/variables.txt") + def _clear_extract_env(monkeypatch): """Ensure EXTRACT/EXTRACT_DATA_PATH do not leak from the suite env.""" monkeypatch.delenv("EXTRACT", raising=False) monkeypatch.delenv("EXTRACT_DATA_PATH", raising=False) + def test_create_request_default_extract(monkeypatch): """EXTRACT default (True) - no skip_extract, root_data_dir unchanged.""" _clear_extract_env(monkeypatch) @@ -87,9 +90,10 @@ def test_create_request_default_extract(monkeypatch): assert actual == expected + def test_create_request_extract_false_with_path(monkeypatch): - """EXTRACT=False + EXTRACT_DATA_PATH set → - skip_extract + override root_data_dir.""" + """EXTRACT=False + EXTRACT_DATA_PATH set → + skip_extract + override root_data_dir.""" _clear_extract_env(monkeypatch) _set_base_env(monkeypatch) monkeypatch.setenv("EXTRACT", "false") @@ -110,6 +114,7 @@ def test_create_request_extract_false_with_path(monkeypatch): assert actual["common"]["root_proc_dir"] == "/path/to/proc/dir/" assert actual["conversion"]["mip_convert_plugin"] == "UKESM1" + def test_create_request_extract_false_without_path_raises(monkeypatch): """EXTRACT=False with no EXTRACT_DATA_PATH → fail with ValueError.""" _clear_extract_env(monkeypatch) @@ -119,4 +124,3 @@ def test_create_request_extract_false_without_path_raises(monkeypatch): with pytest.raises(ValueError, match="EXTRACT=False"): create_request() - From 7f920f6a6b21a766cfd0293374832cc8b882f385 Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Fri, 19 Dec 2025 11:24:15 +0000 Subject: [PATCH 4/7] Changed Copyright dates --- CMEW/app/configure_for/rose-app.conf | 2 +- CMEW/app/configure_standardise/rose-app.conf | 2 +- CMEW/app/housekeeping/rose-app.conf | 2 +- CMEW/app/standardise_model_data/rose-app.conf | 2 +- CMEW/app/unittest/rose-app.conf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index 56705c8f..efcd48a4 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2022-2025, Met Office. +# (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/configure_standardise/rose-app.conf b/CMEW/app/configure_standardise/rose-app.conf index a794e645..ecd720ef 100644 --- a/CMEW/app/configure_standardise/rose-app.conf +++ b/CMEW/app/configure_standardise/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2022-2025, Met Office. +# (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/housekeeping/rose-app.conf b/CMEW/app/housekeeping/rose-app.conf index 8e21fed5..6b2bb74c 100644 --- a/CMEW/app/housekeeping/rose-app.conf +++ b/CMEW/app/housekeeping/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2022-2025, Met Office. +# (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. meta=rose_prune diff --git a/CMEW/app/standardise_model_data/rose-app.conf b/CMEW/app/standardise_model_data/rose-app.conf index 154d4830..12fe8324 100644 --- a/CMEW/app/standardise_model_data/rose-app.conf +++ b/CMEW/app/standardise_model_data/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2022-2025, Met Office. +# (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] diff --git a/CMEW/app/unittest/rose-app.conf b/CMEW/app/unittest/rose-app.conf index 591a560b..e81f016b 100644 --- a/CMEW/app/unittest/rose-app.conf +++ b/CMEW/app/unittest/rose-app.conf @@ -1,4 +1,4 @@ -# (C) Crown Copyright 2022-2025, Met Office. +# (C) Crown Copyright 2024-2025, Met Office. # The LICENSE.md file contains full licensing details. [command] From cab2c219b75e00513bc1788e73aa38d516b0f8b5 Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Fri, 9 Jan 2026 16:05:22 +0000 Subject: [PATCH 5/7] unittest ci --- .github/workflows/default.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml index 6989f722..26cdcb6a 100644 --- a/.github/workflows/default.yml +++ b/.github/workflows/default.yml @@ -42,6 +42,11 @@ jobs: cd CMEW cylc validate -O metoffice . + - name: Run Cylc unit tests + run: | + cd CMEW + conda run -n cmew cylc vip -O metoffice -O unittest . + - name: Run Cylc configuration linter run: | eval "$(conda shell.bash hook)" From 3400be85b907e347c3d7e9c900d76a575441642d Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Tue, 13 Jan 2026 20:15:50 +0000 Subject: [PATCH 6/7] some more config for cdds-extract --- CMEW/flow.cylc | 6 +++++- CMEW/meta/rose-meta.conf | 24 ++++++++++++++++++++++++ CMEW/rose-suite.conf | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index bb88a803..b614c2c7 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -22,7 +22,11 @@ configure_for => configure_standardise configure_recipe => run_recipe configure_standardise => standardise_model_data - standardise_model_data => run_recipe & housekeeping + {%- if HOUSEKEEPING in (True, "true", "True") %} + standardise_model_data => run_recipe & housekeeping + {%- else %} + standardise_model_data => run_recipe + {%- endif %} {%- if TEST %} run_recipe => compare {%- endif %} diff --git a/CMEW/meta/rose-meta.conf b/CMEW/meta/rose-meta.conf index 788425ce..2b79c2f4 100644 --- a/CMEW/meta/rose-meta.conf +++ b/CMEW/meta/rose-meta.conf @@ -290,3 +290,27 @@ help=Must adhere to CMIP6 variant label format: ripf. pattern=^"r[0-9]+i[0-9]+p[0-9]+f[0-9]+"$ sort-key=04 type=quoted + +[template variables=EXTRACT] +compulsory=true +description=To switch OFF or ON CDDS extract +help=The default is true, so data are extracted from MASS prior to a run. +sort-key=17 +type=boolean + +[template variables=EXTRACT_DATA_PATH] +compulsory=true +description=The path to extracted data already available. +help=The default is "", so data to be extracted from MASS. If "", EXTRACT + =must be set to true, otherwise an Error will be raised. +sort-key=18 +type=quoted + +[template variables=HOUSEKEEPING] +compulsory=true +description=To switch OFF or ON the housekeeping task. +help=The default is true, so the locally staged CDDS workspace is removed. + =Make sure you set it to false if you want to keep the data and run + =with EXTRACT=true +sort-key=16 +type=boolean diff --git a/CMEW/rose-suite.conf b/CMEW/rose-suite.conf index fce0a42a..2e65f8bd 100644 --- a/CMEW/rose-suite.conf +++ b/CMEW/rose-suite.conf @@ -16,6 +16,7 @@ DRS_OBS6="default" ESMVALTOOL_MODULE_NAME="" EXTRACT=true EXTRACT_DATA_PATH="" +HOUSEKEEPING=true LABEL_FOR_PLOTS="UKESM1.0 N96ORCA1" MAX_PARALLEL_TASKS=4 MODEL_ID="UKESM1-0-LL" From 8e0473620ebb04f6f10a397c6053a3e0fee78059 Mon Sep 17 00:00:00 2001 From: Nikos Baltas Date: Tue, 13 Jan 2026 22:37:53 +0000 Subject: [PATCH 7/7] re-dumped for ci --- CMEW/meta/rose-meta.conf | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/CMEW/meta/rose-meta.conf b/CMEW/meta/rose-meta.conf index 2b79c2f4..908bd1a7 100644 --- a/CMEW/meta/rose-meta.conf +++ b/CMEW/meta/rose-meta.conf @@ -104,6 +104,30 @@ help=If required, this value must be set in a site-specific configuration file =empty string. type=quoted +[template variables=EXTRACT] +compulsory=true +description=To switch OFF or ON CDDS extract +help=The default is true, so data are extracted from MASS prior to a run. +sort-key=17 +type=boolean + +[template variables=EXTRACT_DATA_PATH] +compulsory=true +description=The path to extracted data already available. +help=The default is "", so data to be extracted from MASS. If "", EXTRACT + =must be set to true, otherwise an Error will be raised. +sort-key=18 +type=quoted + +[template variables=HOUSEKEEPING] +compulsory=true +description=To switch OFF or ON the housekeeping task. +help=The default is true, so the locally staged CDDS workspace is removed. + =Make sure you set it to false if you want to keep the data and run + =with EXTRACT=true +sort-key=16 +type=boolean + [template variables=LABEL_FOR_PLOTS] compulsory=false description=An 'alias' for the evaluation dataset in the ESMValTool recipe. @@ -290,27 +314,3 @@ help=Must adhere to CMIP6 variant label format: ripf. pattern=^"r[0-9]+i[0-9]+p[0-9]+f[0-9]+"$ sort-key=04 type=quoted - -[template variables=EXTRACT] -compulsory=true -description=To switch OFF or ON CDDS extract -help=The default is true, so data are extracted from MASS prior to a run. -sort-key=17 -type=boolean - -[template variables=EXTRACT_DATA_PATH] -compulsory=true -description=The path to extracted data already available. -help=The default is "", so data to be extracted from MASS. If "", EXTRACT - =must be set to true, otherwise an Error will be raised. -sort-key=18 -type=quoted - -[template variables=HOUSEKEEPING] -compulsory=true -description=To switch OFF or ON the housekeeping task. -help=The default is true, so the locally staged CDDS workspace is removed. - =Make sure you set it to false if you want to keep the data and run - =with EXTRACT=true -sort-key=16 -type=boolean