Skip to content

Commit eacf0bb

Browse files
xrotwangxrotwang
committed
add datetime datatype
1 parent 741af20 commit eacf0bb

File tree

9 files changed

+167
-82
lines changed

9 files changed

+167
-82
lines changed

‎csvw/datatype/anyuri.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@ package datatype
22

33
import "net/url"
44

5-
var AnyURI = BaseType{
6-
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
7-
return map[string]any{}, nil
8-
},
5+
var AnyURI = baseType{
6+
GetDerivedDescription: zeroGetDerivedDescription,
7+
SetValueConstraints: zeroSetValueConstraints,
98
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
109
u, err := url.Parse(s)
1110
if err != nil {

‎csvw/datatype/boolean.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"strings"
77
)
88

9-
var Boolean = BaseType{
9+
var Boolean = baseType{
1010
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
1111
val, ok := dtProps["format"]
1212
if ok {
@@ -15,6 +15,7 @@ var Boolean = BaseType{
1515
}
1616
return map[string]any{"true": []string{"true", "1"}, "false": []string{"false", "0"}}, nil
1717
},
18+
SetValueConstraints: zeroSetValueConstraints,
1819
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
1920
if slices.Contains(dt.DerivedDescription["true"].([]string), s) {
2021
return true, nil

‎csvw/datatype/datatype.go‎

Lines changed: 68 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ The CSVW spec defines the following steps to parse a string value in a csv cell:
2828
package datatype
2929

3030
import (
31+
"fmt"
3132
"gocldf/internal/jsonutil"
32-
"strconv"
3333
)
3434

3535
/*
36-
BaseType ties together functions related to converting between string and Go representations of CSVW datatypes.
36+
baseType ties together functions related to converting between string and Go representations of CSVW datatypes.
3737
3838
GetDerivedDescription is called when instantiating a Datatype object.
3939
The result is stored as DerivedDescription member of the Datatype and can be
@@ -49,25 +49,41 @@ SqlType specifies the best matching SQLite data type.
4949
ToSql implements the conversion of the Go object to a suitable object for insertion
5050
into a SQLite database.
5151
*/
52-
type BaseType struct {
52+
type baseType struct {
5353
GetDerivedDescription func(map[string]any) (map[string]any, error)
54+
SetValueConstraints func(map[string]stringAndAny) error
5455
ToGo func(*Datatype, string, bool) (any, error)
5556
ToString func(*Datatype, any) (string, error)
5657
SqlType string
5758
ToSql func(*Datatype, any) (any, error)
5859
}
5960

60-
// BaseTypes provides a mapping of CSVW data type base names to BaseType instances.
61-
var BaseTypes = map[string]BaseType{
62-
"boolean": Boolean,
63-
"string": String,
64-
"anyURI": AnyURI,
65-
"integer": Integer,
66-
"decimal": Decimal,
67-
"float": Decimal,
68-
"number": Decimal,
69-
"double": Decimal,
70-
"json": Json,
61+
func zeroGetDerivedDescription(m map[string]any) (map[string]any, error) {
62+
if len(m) < 0 {
63+
return nil, fmt.Errorf("zeroGetDerivedDescription called with %d values", len(m))
64+
}
65+
return map[string]any{}, nil
66+
}
67+
68+
func zeroSetValueConstraints(m map[string]stringAndAny) error {
69+
if len(m) != 4 {
70+
return fmt.Errorf("zeroGetDerivedDescription called with %d values", len(m))
71+
}
72+
return nil
73+
}
74+
75+
// BaseTypes provides a mapping of CSVW data type base names to baseType instances.
76+
var BaseTypes = map[string]baseType{
77+
"boolean": Boolean,
78+
"string": String,
79+
"anyURI": AnyURI,
80+
"integer": Integer,
81+
"decimal": Decimal,
82+
"float": Decimal,
83+
"number": Decimal,
84+
"double": Decimal,
85+
"json": Json,
86+
"datetime": Datetime,
7187
}
7288

7389
// Datatype holds the data related to a CSVW datatype description.
@@ -86,20 +102,17 @@ type Datatype struct {
86102
DerivedDescription map[string]any
87103
}
88104

105+
type stringAndAny struct {
106+
str string
107+
val any
108+
}
109+
89110
// New is a factory function to create a Datatype as specified in a JSON description.
90111
func New(jsonCol map[string]interface{}) (*Datatype, error) {
91112
var (
113+
s2a stringAndAny
92114
s string
93115
err error
94-
//
95-
minInclusiveS string
96-
maxInclusiveS string
97-
minExclusiveS string
98-
maxExclusiveS string
99-
minInclusive any = nil
100-
maxInclusive any = nil
101-
minExclusive any = nil
102-
maxExclusive any = nil
103116
// We seed the three length constraints with a sentinel value.
104117
length = -1
105118
minLength = -1
@@ -108,6 +121,12 @@ func New(jsonCol map[string]interface{}) (*Datatype, error) {
108121
)
109122
dtProps := map[string]any{}
110123

124+
valueConstraintNames := []string{"minInclusive", "maxInclusive", "minExclusive", "maxExclusive"}
125+
valueConstraints := make(map[string]stringAndAny, 4)
126+
for _, v := range valueConstraintNames {
127+
valueConstraints[v] = stringAndAny{"", nil}
128+
}
129+
111130
val, ok := jsonCol["datatype"]
112131
if ok {
113132
s, ok = val.(string)
@@ -129,64 +148,40 @@ func New(jsonCol map[string]interface{}) (*Datatype, error) {
129148
return nil, err
130149
}
131150
// For constraints which must match the base type we first get the string representation.
132-
minInclusiveS, err = jsonutil.GetString(dtProps, "minimum", "")
133-
if err != nil {
134-
return nil, err
135-
}
136-
maxInclusiveS, err = jsonutil.GetString(dtProps, "maximum", "")
137-
if err != nil {
138-
return nil, err
151+
for _, v := range valueConstraintNames {
152+
s2a = valueConstraints[v]
153+
if s2a.str == "" {
154+
s2a.str, err = jsonutil.GetString(dtProps, v, "")
155+
if err != nil {
156+
return nil, err
157+
}
158+
}
159+
valueConstraints[v] = s2a
139160
}
140-
if minInclusiveS == "" {
141-
minInclusiveS, err = jsonutil.GetString(dtProps, "minInclusive", "")
161+
// minimum is just an alias for minInclusive.
162+
if valueConstraints["minInclusive"].str == "" {
163+
s2a = valueConstraints["minInclusive"]
164+
s2a.str, err = jsonutil.GetString(dtProps, "minimum", "")
142165
if err != nil {
143166
return nil, err
144167
}
168+
valueConstraints["minInclusive"] = s2a
145169
}
146-
minExclusiveS, err = jsonutil.GetString(dtProps, "minExclusive", "")
147-
if err != nil {
148-
return nil, err
149-
}
150-
if maxInclusiveS == "" {
151-
maxInclusiveS, err = jsonutil.GetString(dtProps, "maxInclusive", "")
170+
// and so is maximum
171+
if valueConstraints["maxInclusive"].str == "" {
172+
s2a = valueConstraints["maxInclusive"]
173+
s2a.str, err = jsonutil.GetString(dtProps, "maximum", "")
152174
if err != nil {
153175
return nil, err
154176
}
155-
}
156-
maxExclusiveS, err = jsonutil.GetString(dtProps, "maxExclusive", "")
157-
if err != nil {
158-
return nil, err
177+
valueConstraints["maxInclusive"] = s2a
159178
}
160179
}
161180
}
162181
// Now convert the constraints appropriately:
163-
switch base {
164-
case "integer":
165-
if minInclusiveS != "" {
166-
minInclusive, err = strconv.Atoi(minInclusiveS)
167-
}
168-
if minExclusiveS != "" {
169-
minExclusive, err = strconv.Atoi(minExclusiveS)
170-
}
171-
if maxInclusiveS != "" {
172-
maxInclusive, err = strconv.Atoi(maxInclusiveS)
173-
}
174-
if maxExclusiveS != "" {
175-
maxExclusive, err = strconv.Atoi(maxExclusiveS)
176-
}
177-
case "decimal", "float", "number", "double":
178-
if minInclusiveS != "" {
179-
minInclusive, err = strconv.ParseFloat(minInclusiveS, 64)
180-
}
181-
if minExclusiveS != "" {
182-
minExclusive, err = strconv.ParseFloat(minExclusiveS, 64)
183-
}
184-
if maxInclusiveS != "" {
185-
maxInclusive, err = strconv.ParseFloat(maxInclusiveS, 64)
186-
}
187-
if maxExclusiveS != "" {
188-
maxExclusive, err = strconv.ParseFloat(maxExclusiveS, 64)
189-
}
182+
err = BaseTypes[base].SetValueConstraints(valueConstraints)
183+
if err != nil {
184+
return nil, err
190185
}
191186
// We compute the derived description map once at instantiation.
192187
dd, err := BaseTypes[base].GetDerivedDescription(dtProps)
@@ -200,10 +195,10 @@ func New(jsonCol map[string]interface{}) (*Datatype, error) {
200195
Length: length,
201196
MinLength: minLength,
202197
MaxLength: maxLength,
203-
MinInclusive: minInclusive,
204-
MinExclusive: minExclusive,
205-
MaxInclusive: maxInclusive,
206-
MaxExclusive: maxExclusive,
198+
MinInclusive: valueConstraints["minInclusive"].val,
199+
MinExclusive: valueConstraints["minExclusive"].val,
200+
MaxInclusive: valueConstraints["maxInclusive"].val,
201+
MaxExclusive: valueConstraints["maxExclusive"].val,
207202
}
208203
return res, nil
209204
}

‎csvw/datatype/datatype_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ func TestDatatype_ToGoError(t *testing.T) {
107107
{`{"base":"decimal","minimum":-2.2}`, "-2.3"},
108108
{`{"base":"decimal","minInclusive":-2.2}`, "-2.3"},
109109
{`{"base":"decimal","minExclusive":"0"}`, "0"},
110+
{`{"base":"datetime","maximum":"2018-12-10T20:20:20"}`, "2019-12-10T20:20:20"},
110111
{`{"base":"integer","maxExclusive":"5"}`, "5"},
111112
{`{"base":"string","length":3}`, "ab"},
112113
{`{"base":"string","minLength":3}`, "ab"},
@@ -136,6 +137,8 @@ func TestDatatype_RoundtripValue(t *testing.T) {
136137
{`{"base":"json"}`, `{"k":5}`},
137138
{`{"base":"string"}`, "äöü"},
138139
{`{"base":"anyURI"}`, "http://example.org"},
140+
{`{"base":"datetime"}`, "2018-12-10T20:20:20"},
141+
{`{"base":"datetime","format":"yyyy-MM-ddTHH:mm"}`, "2018-12-10T20:20"},
139142
}
140143
for _, tt := range tests {
141144
t.Run("Roundtrip", func(t *testing.T) {

‎csvw/datatype/datetime.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package datatype
2+
3+
import (
4+
"errors"
5+
"time"
6+
)
7+
8+
var (
9+
ISO8061Layout = "2006-01-02T15:04:05"
10+
FormatToLayout = map[string]string{
11+
//with one or more trailing S characters indicating the maximum number of fractional seconds e.g., yyyy-MM-ddTHH:mm:ss.SSS for 2015-03-15T15:02:37.143
12+
"yyyy-MM-ddTHH:mm:ss.S": "2015-03-15T15:02:37.1",
13+
"yyyy-MM-ddTHH:mm:ss": "2015-03-15T15:02:37",
14+
"yyyy-MM-ddTHH:mm": "2015-03-15T15:02",
15+
// any of the date formats above, followed by a single space,
16+
// followed by any of the time formats above, e.g., M/d/yyyy HH:mm
17+
// for 3/22/2015 15:02 or dd.MM.yyyy HH:mm:ss for 22.03.2015 15:02:37
18+
}
19+
)
20+
21+
var Datetime = baseType{
22+
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
23+
val, ok := dtProps["format"]
24+
if ok {
25+
// FIXME: must make sure regex is anchored on both sides! I.e. wrap in "^$" if necessary.
26+
return map[string]any{"layout": FormatToLayout[val.(string)]}, nil
27+
}
28+
return map[string]any{"layout": ISO8061Layout}, nil
29+
},
30+
SetValueConstraints: func(m map[string]stringAndAny) (err error) {
31+
for k, v := range m {
32+
if v.str != "" {
33+
v.val, err = time.Parse(ISO8061Layout, v.str)
34+
}
35+
m[k] = v
36+
}
37+
return
38+
},
39+
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
40+
val, err := time.Parse(dt.DerivedDescription["layout"].(string), s)
41+
if err != nil {
42+
return nil, err
43+
}
44+
if !noChecks {
45+
if dt.MinInclusive != nil && val.Before(dt.MinInclusive.(time.Time)) {
46+
return nil, errors.New("value smaller than minimum")
47+
}
48+
if dt.MaxInclusive != nil && val.After(dt.MaxInclusive.(time.Time)) {
49+
return nil, errors.New("value greater than maximum")
50+
}
51+
if dt.MinExclusive != nil && (val.Equal(dt.MaxInclusive.(time.Time)) || val.Before(dt.MaxInclusive.(time.Time))) {
52+
return nil, errors.New("value smaller than exclusive minimum")
53+
}
54+
if dt.MaxExclusive != nil && (val.Equal(dt.MaxInclusive.(time.Time)) || val.After(dt.MaxInclusive.(time.Time))) {
55+
return nil, errors.New("value greater than exclusive maximum")
56+
}
57+
}
58+
return val, nil
59+
},
60+
ToString: func(dt *Datatype, x any) (string, error) {
61+
return x.(time.Time).Format(dt.DerivedDescription["layout"].(string)), nil
62+
},
63+
SqlType: "TEXT",
64+
ToSql: func(dt *Datatype, x any) (any, error) {
65+
return x.(time.Time).Format(dt.DerivedDescription["layout"].(string)), nil
66+
},
67+
}

‎csvw/datatype/decimal.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,19 @@ import (
66
"strconv"
77
)
88

9-
var Decimal = BaseType{
9+
var Decimal = baseType{
1010
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
1111
return map[string]any{}, nil
1212
},
13+
SetValueConstraints: func(m map[string]stringAndAny) (err error) {
14+
for k, v := range m {
15+
if v.str != "" {
16+
v.val, err = strconv.ParseFloat(v.str, 64)
17+
}
18+
m[k] = v
19+
}
20+
return
21+
},
1322
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
1423
val, err := strconv.ParseFloat(s, 64)
1524
if err != nil {

‎csvw/datatype/integer.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,19 @@ import (
55
"strconv"
66
)
77

8-
var Integer = BaseType{
8+
var Integer = baseType{
99
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
1010
return map[string]any{}, nil
1111
},
12+
SetValueConstraints: func(m map[string]stringAndAny) (err error) {
13+
for k, v := range m {
14+
if v.str != "" {
15+
v.val, err = strconv.Atoi(v.str)
16+
}
17+
m[k] = v
18+
}
19+
return
20+
},
1221
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
1322
val, err := strconv.Atoi(s)
1423
if err != nil {

‎csvw/datatype/json.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ package datatype
22

33
import "encoding/json"
44

5-
var Json = BaseType{
5+
var Json = baseType{
66
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
77
return map[string]any{}, nil
88
},
9+
SetValueConstraints: zeroSetValueConstraints,
910
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
1011
var result any
1112
err := json.Unmarshal([]byte(s), &result)

‎csvw/datatype/string.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"regexp"
66
)
77

8-
var String = BaseType{
8+
var String = baseType{
99
GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
1010
val, ok := dtProps["format"]
1111
if ok {
@@ -14,6 +14,7 @@ var String = BaseType{
1414
}
1515
return map[string]any{"regex": nil}, nil
1616
},
17+
SetValueConstraints: zeroSetValueConstraints,
1718
ToGo: func(dt *Datatype, s string, noChecks bool) (any, error) {
1819
if !noChecks {
1920
if dt.Length != -1 && len(s) != dt.Length {

0 commit comments

Comments
 (0)