Skip to content

Commit 433c87d

Browse files
alexanderkielalexanderkiel
committed
[WIP] Implement Index Garbage Collection
1 parent 3aeb70d commit 433c87d

File tree

38 files changed

+2213
-59
lines changed

38 files changed

+2213
-59
lines changed

‎.github/scripts/admin-api/run-prune-job.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash -e
2+
3+
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
4+
. "$SCRIPT_DIR/../util.sh"
5+
6+
BASE="http://localhost:8080/fhir"
7+
8+
prune-job() {
9+
cat <<END
10+
{
11+
"resourceType": "Task",
12+
"id": "PruneJobReadyExample",
13+
"meta": {
14+
"profile": [
15+
"https://samply.github.io/blaze/fhir/StructureDefinition/PruneJob"
16+
]
17+
},
18+
"input": [
19+
{
20+
"type": {
21+
"coding": [
22+
{
23+
"code": "t",
24+
"system": "https://samply.github.io/blaze/fhir/CodeSystem/PruneJobParameter",
25+
"display": "T"
26+
}
27+
]
28+
},
29+
"valuePositiveInt": 1000
30+
}
31+
],
32+
"code": {
33+
"coding": [
34+
{
35+
"code": "prune",
36+
"system": "https://samply.github.io/blaze/fhir/CodeSystem/JobType",
37+
"display": "Prune the Database"
38+
}
39+
]
40+
},
41+
"status": "ready",
42+
"intent": "order",
43+
"authoredOn": "2024-10-15T15:01:00.000Z"
44+
}
45+
END
46+
}
47+
48+
RESULT="$(curl -s -H 'Content-Type: application/fhir+json' -H 'Accept: application/fhir+json' -d "$(prune-job)" "$BASE/__admin/Task")"
49+
test "resource type" "$(echo "$RESULT" | jq -r .resourceType)" "Task"
50+
test "status" "$(echo "$RESULT" | jq -r .status)" "ready"
51+
52+
sleep 2
53+
54+
ID="$(echo "$RESULT" | jq -r .id)"
55+
RESULT="$(curl -s -H 'Accept: application/fhir+json' "$BASE/__admin/Task/$ID")"
56+
test "resource type" "$(echo "$RESULT" | jq -r .resourceType)" "Task"
57+
test "status" "$(echo "$RESULT" | jq -r .status)" "completed"
58+
59+
echo "$RESULT" | jq '.'

‎.github/scripts/check-referential-integrity-enforced.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
ENFORCED=$(curl -s http://localhost:8080/fhir/metadata | jq -r 'isempty(.rest[].resource[].referencePolicy[] | select(. == "enforced")) | not')
44

55
if [ "true" = "$ENFORCED" ]; then
6-
echo "✅"
6+
echo "✅ Referential Integrity Enforced is enforced"
77
else
88
echo "Fail 😞"
99
exit 1

‎.github/workflows/build.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1204,6 +1204,15 @@ jobs:
12041204
- name: Check Total-Number of Resources are 92114
12051205
run: .github/scripts/check-total-number-of-resources.sh 92114
12061206

1207+
- name: Check Estimate Number of Keys in the ResourceAsOf Index
1208+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "resource-as-of-index" "92114"
1209+
1210+
- name: Check Estimate Number of Keys in the TypeAsOf Index
1211+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "type-as-of-index" "92114"
1212+
1213+
- name: Check Estimate Number of Keys in the SystemAsOf Index
1214+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "system-as-of-index" "92114"
1215+
12071216
- name: Patient Purge all
12081217
run: .github/scripts/patient-purge-all.sh
12091218

@@ -1216,6 +1225,21 @@ jobs:
12161225
- name: Check Resource Totals
12171226
run: .github/scripts/check-resource-totals-after-patient-purge.sh
12181227

1228+
- name: Run Prune Job
1229+
run: .github/scripts/admin-api/run-prune-job.sh
1230+
1231+
- name: Check Total-Number of Resources are 92114
1232+
run: .github/scripts/check-total-number-of-resources.sh 13358
1233+
1234+
- name: Check Estimate Number of Keys in the ResourceAsOf Index
1235+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "resource-as-of-index" "13358"
1236+
1237+
- name: Check Estimate Number of Keys in the TypeAsOf Index
1238+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "type-as-of-index" "13358"
1239+
1240+
- name: Check Estimate Number of Keys in the SystemAsOf Index
1241+
run: .github/scripts/admin-api/check-column-family-estimate-num-keys.sh "system-as-of-index" "13358"
1242+
12191243
- name: Docker Stats
12201244
run: docker stats --no-stream
12211245

‎job-ig/input/fsh/job.fsh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ Id: JobType
88
Title: "Job Type"
99
* ^status = #active
1010
* #re-index "(Re)Index a Search Parameter"
11+
* #prune "Prune the Database"
1112
* #compact "Compact Database Column Families"
1213
* #async-interaction "Asynchronous Interaction Request"
1314
* #async-bulk-data "Asynchronous Bulk Data Request"

‎job-ig/input/fsh/prune-job.fsh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
Alias: UCUM = http://unitsofmeasure.org
2+
Alias: $FT = http://hl7.org/fhir/fhir-types
3+
Alias: $JT = https://samply.github.io/blaze/fhir/CodeSystem/JobType
4+
Alias: $JSR = https://samply.github.io/blaze/fhir/CodeSystem/JobStatusReason
5+
Alias: $JO = https://samply.github.io/blaze/fhir/CodeSystem/JobOutput
6+
Alias: $PJP = https://samply.github.io/blaze/fhir/CodeSystem/PruneJobParameter
7+
Alias: $PJO = https://samply.github.io/blaze/fhir/CodeSystem/PruneJobOutput
8+
Alias: $PI = https://samply.github.io/blaze/fhir/CodeSystem/PruneIndices
9+
10+
CodeSystem: PruneJobParameter
11+
Id: PruneJobParameter
12+
Title: "Prune Job Parameter"
13+
* ^status = #active
14+
* #t "T"
15+
16+
CodeSystem: PruneJobOutput
17+
Id: PruneJobOutput
18+
Title: "Prune Job Output"
19+
* ^status = #active
20+
* #total-index-entries "Total Index Entries"
21+
* #index-entries-processed "Index Entries Processed"
22+
* #index-entries-deleted "Index Entries Deleted"
23+
* #processing-duration "Processing Duration"
24+
* #next-index "Next Index"
25+
* #next-type "Next Type"
26+
* #next-id "Next Id"
27+
* #next-t "Next T"
28+
29+
CodeSystem: PruneIndices
30+
Id: PruneIndices
31+
Title: "Prune Indices"
32+
* ^status = #active
33+
* #resource-as-of "ResourceAsOf"
34+
* #type-as-of "TypeAsOf"
35+
* #system-as-of "SystemAsOf"
36+
37+
ValueSet: PruneIndices
38+
Id: PruneIndices
39+
Title: "Prune Indices Value Set"
40+
* ^status = #active
41+
* include codes from system PruneIndices
42+
43+
Profile: PruneJob
44+
Parent: Job
45+
* code = $JT#prune "Prune the Database"
46+
* input ^slicing.discriminator.type = #pattern
47+
* input ^slicing.discriminator.path = "type"
48+
* input ^slicing.rules = #open
49+
* input contains t 1..1
50+
* input[t] ^short = "T"
51+
* input[t] ^definition = "The database point in time to use as start of pruning."
52+
* input[t].type = $PJP#t
53+
* input[t].value[x] only positiveInt
54+
* output ^slicing.discriminator.type = #pattern
55+
* output ^slicing.discriminator.path = "type"
56+
* output ^slicing.rules = #open
57+
* output contains totalIndexEntries 0..1
58+
* output[totalIndexEntries] ^short = "Total Index Entries"
59+
* output[totalIndexEntries] ^definition = "Estimated total number of index entries to prune."
60+
* output[totalIndexEntries].type = $PJO#total-index-entries
61+
* output[totalIndexEntries].value[x] only unsignedInt
62+
* output contains indexEntriesProcessed 0..1
63+
* output[indexEntriesProcessed] ^short = "Index Entries Processed"
64+
* output[indexEntriesProcessed] ^definition = "Number of index entries processed."
65+
* output[indexEntriesProcessed].type = $PJO#index-entries-processed
66+
* output[indexEntriesProcessed].value[x] only unsignedInt
67+
* output contains indexEntriesDeleted 0..1
68+
* output[indexEntriesDeleted] ^short = "Index Entries Deleted"
69+
* output[indexEntriesDeleted] ^definition = "Number of index entries deleted."
70+
* output[indexEntriesDeleted].type = $PJO#index-entries-deleted
71+
* output[indexEntriesDeleted].value[x] only unsignedInt
72+
* output contains processingDuration 0..1
73+
* output[processingDuration] ^short = "Processing Duration"
74+
* output[processingDuration] ^definition = "Duration the pruning processing took. Durations while the job was paused don't count."
75+
* output[processingDuration].type = $PJO#processing-duration
76+
* output[processingDuration].value[x] only Quantity
77+
* output[processingDuration].valueQuantity
78+
* system 1..1
79+
* system = UCUM
80+
* code 1..1
81+
* code = #s "seconds"
82+
* output contains nextIndex 0..1
83+
* output[nextIndex] ^short = "Next Index"
84+
* output[nextIndex] ^definition = "The name of the index to continue with. Used in case the job is resumed after manual pausing or shutdown of Blaze."
85+
* output[nextIndex].type = $PJO#next-index
86+
* output[nextIndex].value[x] only code
87+
* output[nextIndex].valueCode from PruneIndices
88+
* output contains nextType 0..1
89+
* output[nextType] ^short = "Next Type"
90+
* output[nextType] ^definition = "The FHIR resource type to continue with. Used in case the job is resumed after manual pausing or shutdown of Blaze."
91+
* output[nextType].type = $PJO#next-type
92+
* output[nextType].value[x] only code
93+
* output[nextType].valueCode from http://hl7.org/fhir/ValueSet/resource-types
94+
* output contains nextId 0..1
95+
* output[nextId] ^short = "Next Id"
96+
* output[nextId] ^definition = "The FHIR resource id to continue with. Used in case the job is resumed after manual pausing or shutdown of Blaze."
97+
* output[nextId].type = $PJO#next-id
98+
* output[nextId].value[x] only id
99+
* output contains nextT 0..1
100+
* output[nextT] ^short = "Next T"
101+
* output[nextT] ^definition = "The database point in time to continue with. Used in case the job is resumed after manual pausing or shutdown of Blaze."
102+
* output[nextT].type = $PJO#next-t
103+
* output[nextT].value[x] only positiveInt
104+
105+
Instance: PruneJobReadyExample
106+
InstanceOf: PruneJob
107+
* status = #ready
108+
* intent = #order
109+
* code = $JT#prune "Prune the Database"
110+
* authoredOn = "2024-10-15T15:01:00.000Z"
111+
* input[t].type = $PJP#t "T"
112+
* input[t].valuePositiveInt = 42
113+
114+
Instance: PruneJobInProgressExample
115+
InstanceOf: PruneJob
116+
* status = #in-progress
117+
* statusReason = $JSR#started "Started"
118+
* intent = #order
119+
* code = $JT#prune "Prune the Database"
120+
* authoredOn = "2024-10-15T15:01:00.000Z"
121+
* input[t].type = $PJP#t "T"
122+
* input[t].valuePositiveInt = 42
123+
* output[totalIndexEntries].type = $PJO#total-index-entries "Total Index Entries"
124+
* output[totalIndexEntries].valueUnsignedInt = 1000
125+
* output[indexEntriesProcessed].type = $PJO#index-entries-processed "Index Entries Processed"
126+
* output[indexEntriesProcessed].valueUnsignedInt = 100
127+
* output[indexEntriesDeleted].type = $PJO#index-entries-deleted "Index Entries Deleted"
128+
* output[indexEntriesDeleted].valueUnsignedInt = 10
129+
* output[processingDuration].type = $PJO#processing-duration "Processing Duration"
130+
* output[processingDuration].valueQuantity.value = 10
131+
* output[nextIndex].type = $PJO#next-index "Next Index"
132+
* output[nextIndex].valueCode = $PI#resource-as-of
133+
* output[nextType].type = $PJO#next-type "Next Type"
134+
* output[nextType].valueCode = $FT#Patient
135+
* output[nextId].type = $PJO#next-id "Next Id"
136+
* output[nextId].valueId = "0"
137+
* output[nextT].type = $PJO#next-t "Next T"
138+
* output[nextT].valuePositiveInt = 23
139+
140+
Instance: PruneJobFailedExample
141+
InstanceOf: PruneJob
142+
* status = #failed
143+
* intent = #order
144+
* code = $JT#prune "Prune the Database"
145+
* authoredOn = "2024-10-15T15:01:00.000Z"
146+
* input[t].type = $PJP#t "T"
147+
* input[t].valuePositiveInt = 42
148+
* output[error].type = $JO#error "Error"
149+
* output[error].valueString = "error message"

0 commit comments

Comments
 (0)