Skip to content

Commit 9bb19c4

Browse files
tomkralidisdavid-i-berry
tomkralidis
and
david-i-berry
authored
make csv2bufr template conformsTo an array for consistency with OGC (#152)
* make csv2bufr template conformsTo an array for consistency with OGC * fix ref * fix ref * allow array or string for backwards compat * ignore flake8 global var warning * fix flake8 * add v3 * Support for v3 of schema in templates added. Warning that support for v2 will be removed. * Support for v3 of schema in templates added. Warning that support for v2 will be removed. * flake8 --------- Co-authored-by: david-i-berry <[email protected]>
1 parent 46a3962 commit 9bb19c4

File tree

7 files changed

+331
-41
lines changed

7 files changed

+331
-41
lines changed

‎.flake8‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[flake8]
2+
ignore = F824

‎csv2bufr/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ def __init__(self, descriptors: list,
361361

362362
def create_template(self) -> None:
363363
template = {}
364-
template["conformsTo"] = "csv2bufr-template-v2.json"
364+
template["conformsTo"] = ["csv2bufr-template-v3.json"]
365365
template["metadata"] = {
366366
"label": "",
367367
"description": "",

‎csv2bufr/templates/__init__.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
THISDIR = os.path.dirname(os.path.realpath(__file__))
3030
LOGGER = logging.getLogger(__name__)
3131
SCHEMA = f"{THISDIR}{os.sep}resources{os.sep}schema"
32+
SCHEMA_VERSIONS = ['csv2bufr-template-v2.json', 'csv2bufr-template-v3.json']
3233
TEMPLATE_DIRS = [] # [Path("./")]
3334

3435
_SUCCESS_ = True
@@ -152,7 +153,7 @@ def validate_template(mapping: dict) -> bool:
152153
:returns: `bool` of validation result
153154
"""
154155
# load internal file schema for mappings
155-
file_schema = f"{SCHEMA}{os.sep}csv2bufr-template-v2.json"
156+
file_schema = f"{SCHEMA}{os.sep}csv2bufr-template-v3.json"
156157
try:
157158
with open(file_schema) as fh:
158159
schema = json.load(fh)
@@ -178,10 +179,17 @@ def index_templates() -> bool:
178179
# check if valid mapping file
179180
with template.open() as fh:
180181
tmpl = json.load(fh)
181-
if 'csv2bufr-template-v2.json' not in tmpl.get("conformsTo",[]): # noqa
182-
LOGGER.warning("'csv2bufr-template-v2.json' not found in " + # noqa
183-
f"conformsTo for file {template}, skipping") # noqa
182+
valid_schema = False
183+
for sc in SCHEMA_VERSIONS:
184+
if sc in tmpl.get("conformsTo", []):
185+
valid_schema = True
186+
break
187+
if not valid_schema:
188+
LOGGER.warning(
189+
"No valid schema not found in " + # noqa
190+
f"conformsTo for file {template}, skipping") # noqa
184191
continue
192+
185193
if validate_template(tmpl) == _SUCCESS_:
186194
# get label if exists else set to empty string
187195
fname = str(template)

‎csv2bufr/templates/resources/schema/csv2bufr-template-v3.json‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
{
2+
"$id": "csv2bufr.wis2.0.node.wis",
3+
"$schema": "https://json-schema.org/draft/2020-12/schema",
4+
"type": "object",
5+
"properties": {
6+
"conformsTo": {
7+
"type": ["array", "string"]
8+
},
9+
"metadata": {
10+
"type": "object",
11+
"required": ["label","description","version","author","editor","dateCreated","dateModified","id"],
12+
"properties": {
13+
"label": {
14+
"type": "string"
15+
},
16+
"description": {
17+
"type": "string"
18+
},
19+
"version": {
20+
"type": "string"
21+
},
22+
"author": {
23+
"type": "string"
24+
},
25+
"editor": {
26+
"type": "string"
27+
},
28+
"dateCreated": {
29+
"type": "string",
30+
"format": "date"
31+
},
32+
"dateModified": {
33+
"type": "string",
34+
"format": "date"
35+
},
36+
"id": {
37+
"type": "string",
38+
"format": "uuid4"
39+
}
40+
}
41+
},
42+
43+
"inputShortDelayedDescriptorReplicationFactor": {
44+
"type": "array",
45+
"items": {"type": "integer"}
46+
},
47+
"inputDelayedDescriptorReplicationFactor": {
48+
"type": "array",
49+
"items": {"type": "integer"}
50+
},
51+
"inputExtendedDelayedDescriptorReplicationFactor": {
52+
"type": "array",
53+
"items": {"type": "integer"}
54+
},
55+
"number_header_rows": {
56+
"type": "integer",
57+
"description": "Number of header rows in file before the data"
58+
},
59+
"column_names_row": {
60+
"type": "integer",
61+
"description": "Which header line the column names is given on"
62+
63+
},
64+
"wigos_station_identifier": {
65+
"type": "string",
66+
"description": "Either the WIGOS station identifier for the data or the column in the CSV file containing the identifier"
67+
},
68+
"delimiter": {
69+
"type": "string",
70+
"description": "The delimiter used to separate fields in the input csv file, must be one of ',', ';'. '|' or [tab]"
71+
},
72+
"quoting": {
73+
"type": "string",
74+
"description": "CSV quoting method to use, must be one of QUOTE_NONNUMERIC, QUOTE_ALL, QUOTE_MINIMAL or QUOTE_NONE"
75+
},
76+
"quotechar": {
77+
"type": "string",
78+
"description": "quote character to use, e.g. \", ' etc"
79+
},
80+
"header":{
81+
"type": "array",
82+
"items": {"$ref": "#/$defs/bufr_element"},
83+
"description": "Contents of header sections of BUFR message"
84+
},
85+
"data": {
86+
"type": "array",
87+
"items": {"$ref": "#/$defs/bufr_element"},
88+
"description": "mapping from CSV file (or metadata json file) to BUFR"
89+
}
90+
},
91+
"required" : [
92+
"conformsTo", "metadata",
93+
"inputShortDelayedDescriptorReplicationFactor",
94+
"inputDelayedDescriptorReplicationFactor",
95+
"inputExtendedDelayedDescriptorReplicationFactor",
96+
"column_names_row","number_header_rows","header","data"],
97+
98+
"$defs":{
99+
"bufr_element": {
100+
"type": "object",
101+
"properties": {
102+
"eccodes_key": {
103+
"type": "string",
104+
"descripition": "eccodes key used to set the value in the BUFR data"
105+
},
106+
"value": {
107+
"type": [
108+
"string"
109+
],
110+
"description": "where to extract the value from, can be one off 'data','metadata','const','array' followed by the value or column header"
111+
},
112+
"valid_min": {
113+
"type": "string",
114+
"description": "Minimum valid value for parameter if set"
115+
},
116+
"valid_max": {
117+
"type": "string",
118+
"description": "Maximum value for for the parameter if set"
119+
},
120+
"scale": {
121+
"type": "string",
122+
"description": "Value used to scale the data by before encoding using the same conventions as in BUFR"
123+
},
124+
"offset": {
125+
"type": "string",
126+
"description": "Value added to the data before encoding to BUFR following the same conventions as BUFR"
127+
}
128+
},
129+
"required": ["eccodes_key", "value"],
130+
"allOf": [
131+
{
132+
"dependentRequired": {"scale": ["offset"]}
133+
},
134+
{
135+
"dependentRequired": {"offset": ["scale"]}
136+
}
137+
]
138+
}
139+
}
140+
}

‎docs/source/conf.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@
7373
html_static_path = ['_static']
7474

7575
html_css_files = [
76-
'wmo.css', # overrides for wide tables in RTD theme
77-
]
76+
'wmo.css', # overrides for wide tables in RTD theme
77+
]
7878

7979
# options for maths
8080
imgmath_image_format = 'svg'

‎docs/source/resources/csv2bufr-template-v3.json‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
{
2+
"$id": "csv2bufr.wis2.0.node.wis",
3+
"$schema": "https://json-schema.org/draft/2020-12/schema",
4+
"type": "object",
5+
"properties": {
6+
"conformsTo": {
7+
"type": ["array", "string"]
8+
},
9+
"metadata": {
10+
"type": "object",
11+
"requiered": ["label","description","version","author","editor","dateCreated","dateModified","id"],
12+
"properties": {
13+
"label": {
14+
"type": "string"
15+
},
16+
"description": {
17+
"type": "string"
18+
},
19+
"version": {
20+
"type": "string"
21+
},
22+
"author": {
23+
"type": "string"
24+
},
25+
"editor": {
26+
"type": "string"
27+
},
28+
"dateCreated": {
29+
"type": "string",
30+
"format": "date"
31+
},
32+
"dateModified": {
33+
"type": "string",
34+
"format": "date"
35+
},
36+
"id": {
37+
"type": "string",
38+
"format": "uuid4"
39+
}
40+
}
41+
},
42+
43+
"inputShortDelayedDescriptorReplicationFactor": {
44+
"type": "array",
45+
"items": {"type": "integer"}
46+
},
47+
"inputDelayedDescriptorReplicationFactor": {
48+
"type": "array",
49+
"items": {"type": "integer"}
50+
},
51+
"inputExtendedDelayedDescriptorReplicationFactor": {
52+
"type": "array",
53+
"items": {"type": "integer"}
54+
},
55+
"number_header_rows": {
56+
"type": "integer",
57+
"description": "Number of header rows in file before the data"
58+
},
59+
"column_names_row": {
60+
"type": "integer",
61+
"description": "Which header line the column names is given on"
62+
63+
},
64+
"wigos_station_identifier": {
65+
"type": "string",
66+
"description": "Either the WIGOS station identifier for the data or the column in the CSV file containing the identifier"
67+
},
68+
"delimiter": {
69+
"type": "string",
70+
"description": "The delimiter used to separate fields in the input csv file, must be one of ',', ';'. '|' or [tab]"
71+
},
72+
"quoting": {
73+
"type": "string",
74+
"description": "CSV quoting method to use, must be one of QUOTE_NONNUMERIC, QUOTE_ALL, QUOTE_MINIMAL or QUOTE_NONE"
75+
},
76+
"quotechar": {
77+
"type": "string",
78+
"description": "quote character to use, e.g. \", ' etc"
79+
},
80+
"header":{
81+
"type": "array",
82+
"items": {"$ref": "#/$defs/bufr_element"},
83+
"description": "Contents of header sections of BUFR message"
84+
},
85+
"data": {
86+
"type": "array",
87+
"items": {"$ref": "#/$defs/bufr_element"},
88+
"description": "mapping from CSV file (or metadata json file) to BUFR"
89+
}
90+
},
91+
"required" : [
92+
"conformsTo", "metadata",
93+
"inputShortDelayedDescriptorReplicationFactor",
94+
"inputDelayedDescriptorReplicationFactor",
95+
"inputExtendedDelayedDescriptorReplicationFactor",
96+
"column_names_row","number_header_rows","header","data"],
97+
98+
"$defs":{
99+
"bufr_element": {
100+
"type": "object",
101+
"properties": {
102+
"eccodes_key": {
103+
"type": "string",
104+
"descripition": "eccodes key used to set the value in the BUFR data"
105+
},
106+
"value": {
107+
"type": [
108+
"string"
109+
],
110+
"description": "where to extract the value from, can be one off 'data','metadata','const','array' followed by the value or column header"
111+
},
112+
"valid_min": {
113+
"type": "string",
114+
"description": "Minimum valid value for parameter if set"
115+
},
116+
"valid_max": {
117+
"type": "string",
118+
"description": "Maximum value for for the parameter if set"
119+
},
120+
"scale": {
121+
"type": "string",
122+
"description": "Value used to scale the data by before encoding using the same conventions as in BUFR"
123+
},
124+
"offset": {
125+
"type": "string",
126+
"description": "Value added to the data before encoding to BUFR following the same conventions as BUFR"
127+
}
128+
},
129+
"required": ["eccodes_key", "value"],
130+
"allOf": [
131+
{
132+
"dependentRequired": {"scale": ["offset"]}
133+
},
134+
{
135+
"dependentRequired": {"offset": ["scale"]}
136+
}
137+
]
138+
}
139+
}
140+
}

0 commit comments

Comments
 (0)