Skip to content

Commit 8a88279

Browse files
nbroylesnbroyles
committed
[dbnode] Remove namespaces from example config and integration tests (#2866)
1 parent 345a67e commit 8a88279

File tree

33 files changed

+270
-241
lines changed

33 files changed

+270
-241
lines changed

‎scripts/docker-integration-tests/aggregator/m3coordinator.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,7 @@ carbon:
1212
retention: 6h
1313

1414
clusters:
15-
- namespaces:
16-
- namespace: agg
17-
type: aggregated
18-
resolution: 10s
19-
retention: 6h
20-
- namespace: unagg
21-
type: unaggregated
22-
retention: 1s
23-
client:
15+
- client:
2416
config:
2517
service:
2618
env: default_env

‎scripts/docker-integration-tests/aggregator/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ function defer {
2929
trap defer EXIT
3030

3131
echo "Setup DB node"
32-
setup_single_m3db_node
32+
AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node
3333

3434
echo "Initializing aggregator topology"
3535
curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{

‎scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,7 @@ carbon:
1212
retention: 6h
1313

1414
clusters:
15-
- namespaces:
16-
- namespace: agg
17-
type: aggregated
18-
resolution: 10s
19-
retention: 6h
20-
- namespace: unagg
21-
type: unaggregated
22-
retention: 1s
23-
client:
15+
- client:
2416
config:
2517
service:
2618
env: default_env

‎scripts/docker-integration-tests/aggregator_legacy/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ function defer {
2222
trap defer EXIT
2323

2424
echo "Setup DB node"
25-
setup_single_m3db_node
25+
AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node
2626

2727
echo "Initializing aggregator topology"
2828
curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{

‎scripts/docker-integration-tests/carbon/m3coordinator.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
11
clusters:
2-
- namespaces:
3-
- namespace: agg
4-
type: aggregated
5-
retention: 10h
6-
resolution: 5s
7-
- namespace: unagg
8-
type: unaggregated
9-
retention: 10m
10-
client:
2+
- client:
113
config:
124
service:
135
env: default_env

‎scripts/docker-integration-tests/carbon/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ function defer {
1919
}
2020
trap defer EXIT
2121

22-
setup_single_m3db_node
22+
AGG_RESOLUTION=5s setup_single_m3db_node
2323

2424
function read_carbon {
2525
target=$1

‎scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,7 @@ limits:
33
maxFetchedSeries: 100
44

55
clusters:
6-
- namespaces:
7-
- namespace: agg
8-
type: aggregated
9-
retention: 10h
10-
resolution: 15s
11-
- namespace: unagg
12-
type: unaggregated
13-
retention: 10h
14-
client:
6+
- client:
157
config:
168
service:
179
env: default_env

‎scripts/docker-integration-tests/common.sh‎

Lines changed: 102 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,32 @@ function setup_single_m3db_node_long_namespaces {
7474
]
7575
}'
7676

77+
echo "Updating aggregation options for agg namespace"
78+
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
79+
"name": "agg",
80+
"options": {
81+
"aggregationOptions": {
82+
"aggregations": [
83+
{
84+
"aggregated": true,
85+
"attributes": {
86+
"resolutionDuration": "30s",
87+
"downsampleOptions": { "all": false }
88+
}
89+
}
90+
]
91+
}
92+
}
93+
}'
94+
7795
echo "Wait until placement is init'd"
7896
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
7997
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
8098

99+
echo "Wait until agg namespace is ready"
100+
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
101+
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
102+
81103
wait_for_namespaces
82104

83105
echo "Adding agg2d namespace"
@@ -86,10 +108,31 @@ function setup_single_m3db_node_long_namespaces {
86108
"retentionTime": "48h"
87109
}'
88110

111+
echo "Updating aggregation options for agg namespace"
112+
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
113+
"name": "agg2d",
114+
"options": {
115+
"aggregationOptions": {
116+
"aggregations": [
117+
{
118+
"aggregated": true,
119+
"attributes": {
120+
"resolutionDuration": "1m",
121+
"downsampleOptions": { "all": false }
122+
}
123+
}
124+
]
125+
}
126+
}
127+
}'
128+
89129
echo "Wait until agg2d namespace is init'd"
90130
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
91131
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]'
92132

133+
echo "Wait until agg2d namespace is ready"
134+
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
135+
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg2d\"}" | grep -c true)" -eq 1 ]'
93136

94137
echo "Wait until bootstrapped"
95138
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
@@ -103,6 +146,8 @@ function setup_single_m3db_node {
103146
local dbnode_id=${DBNODE_ID:-m3db_local}
104147
local coordinator_port=${COORDINATOR_PORT:-7201}
105148
local zone=${ZONE:-embedded}
149+
local agg_resolution=${AGG_RESOLUTION:-15s}
150+
local agg_retention=${AGG_RETENTION:-10h}
106151

107152
echo "Wait for API to be available"
108153
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
@@ -112,7 +157,7 @@ function setup_single_m3db_node {
112157
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
113158
"type": "cluster",
114159
"namespaceName": "agg",
115-
"retentionTime": "6h",
160+
"retentionTime": "'${agg_retention}'",
116161
"num_shards": 4,
117162
"replicationFactor": 1,
118163
"hosts": [
@@ -127,10 +172,31 @@ function setup_single_m3db_node {
127172
]
128173
}'
129174

175+
echo "Updating aggregation options for agg namespace"
176+
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
177+
"name": "agg",
178+
"options": {
179+
"aggregationOptions": {
180+
"aggregations": [
181+
{
182+
"aggregated": true,
183+
"attributes": {
184+
"resolutionDuration": "'${agg_resolution}'"
185+
}
186+
}
187+
]
188+
}
189+
}
190+
}'
191+
130192
echo "Wait until placement is init'd"
131193
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
132194
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
133195

196+
echo "Wait until agg namespace is ready"
197+
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
198+
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
199+
134200
wait_for_namespaces
135201

136202
echo "Wait until bootstrapped"
@@ -147,6 +213,8 @@ function setup_two_m3db_nodes {
147213
local dbnode_host_1_health_port=${DBNODE_HEALTH_PORT_01:-9012}
148214
local dbnode_host_2_health_port=${DBNODE_HEALTH_PORT_02:-9022}
149215
local coordinator_port=${COORDINATOR_PORT:-7201}
216+
local agg_resolution=${AGG_RESOLUTION:-15s}
217+
local agg_retention=${AGG_RETENTION:-10h}
150218

151219
echo "Wait for API to be available"
152220
ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
@@ -156,7 +224,7 @@ function setup_two_m3db_nodes {
156224
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
157225
"type": "cluster",
158226
"namespaceName": "agg",
159-
"retentionTime": "6h",
227+
"retentionTime": "'${agg_retention}'",
160228
"num_shards": 2,
161229
"replicationFactor": 2,
162230
"hosts": [
@@ -179,10 +247,31 @@ function setup_two_m3db_nodes {
179247
]
180248
}'
181249

250+
echo "Updating aggregation options for agg namespace"
251+
curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
252+
"name": "agg",
253+
"options": {
254+
"aggregationOptions": {
255+
"aggregations": [
256+
{
257+
"aggregated": true,
258+
"attributes": {
259+
"resolutionDuration": "'${agg_resolution}'"
260+
}
261+
}
262+
]
263+
}
264+
}
265+
}'
266+
182267
echo "Wait until placement is init'd"
183268
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
184269
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]'
185270

271+
echo "Wait until agg namespace is ready"
272+
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
273+
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
274+
186275
wait_for_namespaces
187276

188277
echo "Wait until bootstrapped"
@@ -194,6 +283,7 @@ function setup_two_m3db_nodes {
194283

195284
function wait_for_namespaces {
196285
local coordinator_port=${COORDINATOR_PORT:-7201}
286+
local unagg_retention=${UNAGG_RETENTION:-10h}
197287

198288
echo "Wait until agg namespace is init'd"
199289
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
@@ -202,13 +292,17 @@ function wait_for_namespaces {
202292
echo "Adding unagg namespace"
203293
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
204294
"namespaceName": "unagg",
205-
"retentionTime": "6h"
295+
"retentionTime": "'${unagg_retention}'"
206296
}'
207297

208298
echo "Wait until unagg namespace is init'd"
209299
ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
210300
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'
211301

302+
echo "Wait until unagg namespace is ready"
303+
ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff \
304+
'[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"unagg\"}" | grep -c true)" -eq 1 ]'
305+
212306
echo "Adding coldWritesRepairAndNoIndex namespace"
213307
curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
214308
"name": "coldWritesRepairAndNoIndex",
@@ -227,6 +321,11 @@ function wait_for_namespaces {
227321
"bufferPastDuration": "10m",
228322
"blockDataExpiry": true,
229323
"blockDataExpiryAfterNotAccessPeriodDuration": "5m"
324+
},
325+
"aggregationOptions": {
326+
"aggregations": [
327+
{ "aggregated": false }
328+
]
230329
}
231330
}
232331
}'

‎scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,5 @@
11
clusters:
2-
- namespaces:
3-
- namespace: agg
4-
type: aggregated
5-
retention: 24h
6-
resolution: 30s
7-
downsample:
8-
all: false
9-
- namespace: agg2d
10-
type: aggregated
11-
retention: 48h
12-
resolution: 1m
13-
downsample:
14-
all: false
15-
- namespace: unagg
16-
type: unaggregated
17-
retention: 10m
18-
client:
2+
- client:
193
config:
204
service:
215
env: default_env

‎scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,7 @@ limits:
33
maxFetchedSeries: 100
44

55
clusters:
6-
- namespaces:
7-
- namespace: agg
8-
type: aggregated
9-
retention: 10h
10-
resolution: 15s
11-
- namespace: unagg
12-
type: unaggregated
13-
retention: 10h
14-
client:
6+
- client:
157
config:
168
service:
179
env: default_env

0 commit comments

Comments
 (0)