Skip to content

Commit 6c6f2c8

Browse files
xrotwangxrotwang
committed
started work on dataset tests
1 parent 212ef78 commit 6c6f2c8

File tree

13 files changed

+1455
-16
lines changed

13 files changed

+1455
-16
lines changed

‎csvw/dataset/dataset_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package dataset
2+
3+
import (
4+
"database/sql"
5+
"gocldf/db"
6+
"testing"
7+
)
8+
9+
func makeDataset(fname string) *Dataset {
10+
ds, err := New("testdata/" + fname)
11+
if err != nil {
12+
panic(err)
13+
}
14+
return ds
15+
}
16+
17+
func TestDataset_simple(t *testing.T) {
18+
ds := makeDataset("StructureDataset-metadata.json")
19+
if len(ds.Tables) != 4 {
20+
t.Errorf(`problem: %q vs %q`, len(ds.Tables), 4)
21+
}
22+
err := ds.LoadData()
23+
if err != nil {
24+
panic(err)
25+
}
26+
db.WithDatabase(":memory:", func(s *sql.DB) error {
27+
err = db.WithTransaction(s, func(tx *sql.Tx) error {
28+
schema, tableData, err := ds.ToSqlite(tx)
29+
if err != nil {
30+
return err
31+
}
32+
_, err = tx.Exec(schema) // Write the schema ...
33+
if err != nil {
34+
return err
35+
}
36+
for _, tData := range tableData { // ... and the data.
37+
db.BatchInsert(tx, tData.TableName, tData.ColNames, tData.Rows)
38+
}
39+
return nil
40+
})
41+
if err != nil {
42+
panic(err)
43+
}
44+
var countLanguages int
45+
err = db.Query(
46+
s,
47+
"select count(*) from languagetable",
48+
func(rows *sql.Rows) error {
49+
return rows.Scan(&countLanguages)
50+
})
51+
if err != nil {
52+
panic(err)
53+
}
54+
if countLanguages != 29 {
55+
t.Errorf(`problem: %q vs %q`, countLanguages, 29)
56+
}
57+
return nil
58+
}, false)
59+
}

‎csvw/dataset/testdata/StructureDataset-metadata.json‎

Lines changed: 302 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
{
2+
"@context": [
3+
"http://www.w3.org/ns/csvw",
4+
{
5+
"@language": "en"
6+
}
7+
],
8+
"dc:bibliographicCitation": "Peterson, J. (2017). Fitting the pieces together \u2013 Towards a linguistic prehistory of eastern-central South Asia (and beyond). Journal of South Asian Languages and Linguistics, 4(2), pp. 211-257. doi:10.1515/jsall-2017-0008",
9+
"dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#StructureDataset",
10+
"dc:identifier": "https://doi.org/10.1515/jsall-2017-0008",
11+
"dc:source": "sources.bib",
12+
"dc:title": "Towards a linguistic prehistory of eastern-central South Asia",
13+
"dcat:accessURL": "https://github.com/cldf-datasets/petersonsouthasia",
14+
"prov:wasDerivedFrom": [
15+
{
16+
"rdf:type": "prov:Entity",
17+
"dc:title": "Repository",
18+
"rdf:about": "https://github.com/cldf-datasets/petersonsouthasia",
19+
"dc:created": "487b8a3"
20+
},
21+
{
22+
"rdf:type": "prov:Entity",
23+
"dc:title": "Glottolog",
24+
"rdf:about": "https://github.com/glottolog/glottolog",
25+
"dc:created": "v4.1"
26+
}
27+
],
28+
"prov:wasGeneratedBy": [
29+
{
30+
"dc:title": "python",
31+
"dc:description": "3.5.2"
32+
},
33+
{
34+
"dc:title": "python-packages",
35+
"dc:relation": "requirements.txt"
36+
}
37+
],
38+
"rdf:ID": "petersonsouthasia",
39+
"rdf:type": "http://www.w3.org/ns/dcat#Distribution",
40+
"dialect": {
41+
"commentPrefix": null
42+
},
43+
"tables": [
44+
{
45+
"dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#ValueTable",
46+
"dc:extent": 812,
47+
"tableSchema": {
48+
"columns": [
49+
{
50+
"datatype": {
51+
"base": "string",
52+
"format": "[a-zA-Z0-9_\\-]+"
53+
},
54+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#id",
55+
"required": true,
56+
"name": "ID"
57+
},
58+
{
59+
"datatype": "string",
60+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#languageReference",
61+
"required": true,
62+
"name": "Language_ID"
63+
},
64+
{
65+
"datatype": "string",
66+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#parameterReference",
67+
"required": true,
68+
"name": "Parameter_ID"
69+
},
70+
{
71+
"datatype": "string",
72+
"null": [
73+
"?"
74+
],
75+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#value",
76+
"required": false,
77+
"name": "Value"
78+
},
79+
{
80+
"datatype": "string",
81+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#codeReference",
82+
"required": false,
83+
"name": "Code_ID"
84+
},
85+
{
86+
"datatype": "string",
87+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#comment",
88+
"required": false,
89+
"name": "Comment"
90+
},
91+
{
92+
"datatype": "string",
93+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#source",
94+
"required": false,
95+
"separator": ";",
96+
"name": "Source"
97+
}
98+
],
99+
"foreignKeys": [
100+
{
101+
"columnReference": [
102+
"Language_ID"
103+
],
104+
"reference": {
105+
"resource": "languages.csv",
106+
"columnReference": [
107+
"ID"
108+
]
109+
}
110+
},
111+
{
112+
"columnReference": [
113+
"Parameter_ID"
114+
],
115+
"reference": {
116+
"resource": "parameters.csv",
117+
"columnReference": [
118+
"ID"
119+
]
120+
}
121+
},
122+
{
123+
"columnReference": [
124+
"Code_ID"
125+
],
126+
"reference": {
127+
"resource": "codes.csv",
128+
"columnReference": [
129+
"ID"
130+
]
131+
}
132+
}
133+
],
134+
"primaryKey": [
135+
"ID"
136+
]
137+
},
138+
"url": "values.csv"
139+
},
140+
{
141+
"dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#LanguageTable",
142+
"dc:extent": 29,
143+
"tableSchema": {
144+
"columns": [
145+
{
146+
"datatype": {
147+
"base": "string",
148+
"format": "[a-zA-Z0-9_\\-]+"
149+
},
150+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#id",
151+
"required": true,
152+
"name": "ID"
153+
},
154+
{
155+
"datatype": "string",
156+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#name",
157+
"required": false,
158+
"name": "Name"
159+
},
160+
{
161+
"datatype": "string",
162+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#macroarea",
163+
"required": false,
164+
"name": "Macroarea"
165+
},
166+
{
167+
"datatype": {
168+
"base": "decimal",
169+
"minimum": -90,
170+
"maximum": 90
171+
},
172+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#latitude",
173+
"required": false,
174+
"name": "Latitude"
175+
},
176+
{
177+
"datatype": {
178+
"base": "decimal",
179+
"minimum": -180,
180+
"maximum": 180
181+
},
182+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#longitude",
183+
"required": false,
184+
"name": "Longitude"
185+
},
186+
{
187+
"datatype": {
188+
"base": "string",
189+
"format": "[a-z0-9]{4}[1-9][0-9]{3}"
190+
},
191+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#glottocode",
192+
"required": false,
193+
"valueUrl": "http://glottolog.org/resource/languoid/id/{glottocode}",
194+
"name": "Glottocode"
195+
},
196+
{
197+
"datatype": {
198+
"base": "string",
199+
"format": "[a-z]{3}"
200+
},
201+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#iso639P3code",
202+
"required": false,
203+
"name": "ISO639P3code"
204+
},
205+
{
206+
"datatype": "string",
207+
"name": "Family_name"
208+
}
209+
],
210+
"primaryKey": [
211+
"ID"
212+
]
213+
},
214+
"url": "languages.csv"
215+
},
216+
{
217+
"dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#ParameterTable",
218+
"dc:extent": 28,
219+
"tableSchema": {
220+
"columns": [
221+
{
222+
"datatype": {
223+
"base": "string",
224+
"format": "[a-zA-Z0-9_\\-]+"
225+
},
226+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#id",
227+
"required": true,
228+
"name": "ID"
229+
},
230+
{
231+
"datatype": "string",
232+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#name",
233+
"required": false,
234+
"name": "Name"
235+
},
236+
{
237+
"datatype": "string",
238+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#description",
239+
"required": false,
240+
"name": "Description"
241+
}
242+
],
243+
"primaryKey": [
244+
"ID"
245+
]
246+
},
247+
"url": "parameters.csv"
248+
},
249+
{
250+
"dc:conformsTo": "http://cldf.clld.org/v1.0/terms.rdf#CodeTable",
251+
"dc:extent": 75,
252+
"tableSchema": {
253+
"columns": [
254+
{
255+
"datatype": {
256+
"base": "string",
257+
"format": "[a-zA-Z0-9_\\-]+"
258+
},
259+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#id",
260+
"required": true,
261+
"name": "ID"
262+
},
263+
{
264+
"datatype": "string",
265+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#parameterReference",
266+
"required": true,
267+
"name": "Parameter_ID"
268+
},
269+
{
270+
"datatype": "string",
271+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#name",
272+
"required": false,
273+
"name": "Name"
274+
},
275+
{
276+
"datatype": "string",
277+
"propertyUrl": "http://cldf.clld.org/v1.0/terms.rdf#description",
278+
"required": false,
279+
"name": "Description"
280+
}
281+
],
282+
"foreignKeys": [
283+
{
284+
"columnReference": [
285+
"Parameter_ID"
286+
],
287+
"reference": {
288+
"resource": "parameters.csv",
289+
"columnReference": [
290+
"ID"
291+
]
292+
}
293+
}
294+
],
295+
"primaryKey": [
296+
"ID"
297+
]
298+
},
299+
"url": "codes.csv"
300+
}
301+
]
302+
}

0 commit comments

Comments
 (0)