Skip to content

Commit c74b4db

Browse files
Denovo1998Denovo1998
committed
Add support for BookKeeper indexDirectories configuration
1 parent 804192a commit c74b4db

File tree

5 files changed

+140
-0
lines changed

5 files changed

+140
-0
lines changed

‎charts/pulsar/templates/_bookkeeper.tpl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,25 @@ PULSAR_PREFIX_tlsTrustStore: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem"
170170
{{- end }}
171171
{{- end }}
172172

173+
{{/*
174+
Render BookKeeper indexDirectories as comma-separated string.
175+
Accepts either a string or a list of strings.
176+
*/}}
177+
{{- define "pulsar.bookkeeper.indexDirectories" -}}
178+
{{- $v := .Values.bookkeeper.indexDirectories -}}
179+
{{- if $v -}}
180+
{{- if kindIs "string" $v -}}
181+
{{- $v -}}
182+
{{- else -}}
183+
{{- $v | join "," -}}
184+
{{- end -}}
185+
{{- else if and .Values.bookkeeper.volumes.index.enabled .Values.bookkeeper.volumes.index.mountPath -}}
186+
{{- .Values.bookkeeper.volumes.index.mountPath -}}
187+
{{- else -}}
188+
{{- "" -}}
189+
{{- end -}}
190+
{{- end -}}
191+
173192
{{/*
174193
Define bookie init container : verify cluster id
175194
*/}}

‎charts/pulsar/templates/bookkeeper-configmap.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@ data:
5555
{{- else }}
5656
ledgerDirectories: "/pulsar/data/bookkeeper/ledgers"
5757
{{- end }}
58+
{{- $indexDirs := include "pulsar.bookkeeper.indexDirectories" . }}
59+
{{- if $indexDirs }}
60+
# Configure RocksDB/index location for DbLedgerStorage. When unset, BookKeeper falls back to ledgerDirectories.
61+
indexDirectories: {{ $indexDirs | quote }}
62+
PULSAR_PREFIX_indexDirectories: {{ $indexDirs | quote }}
63+
{{- end }}
5864
{{- if .Values.functions.useBookieAsStateStore }}
5965
# Stateful function config
6066
extraServerComponents: "org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent"

‎charts/pulsar/templates/bookkeeper-statefulset.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,18 @@ spec:
212212
nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 &
213213
cd /pulsar;
214214
{{- end }}
215+
{{- $indexDirs := include "pulsar.bookkeeper.indexDirectories" . -}}
216+
{{- if $indexDirs }}
217+
INDEX_DIRS={{ $indexDirs | quote }}
218+
if [ -n "$INDEX_DIRS" ]; then
219+
oldIFS=$IFS
220+
IFS=','
221+
for dir in $INDEX_DIRS; do
222+
mkdir -p "$dir"
223+
done
224+
IFS=$oldIFS
225+
fi
226+
{{- end }}
215227
bin/apply-config-from-env.py conf/bookkeeper.conf;
216228
{{- include "pulsar.bookkeeper.zookeeper.tls.settings" . | nindent 10 }}
217229
OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar bookie;
@@ -227,6 +239,10 @@ spec:
227239
{{- if .Values.bookkeeper.volumes.useSingleCommonVolume }}
228240
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}"
229241
mountPath: /pulsar/data/bookkeeper
242+
{{- if .Values.bookkeeper.volumes.index.enabled }}
243+
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
244+
mountPath: {{ .Values.bookkeeper.volumes.index.mountPath }}
245+
{{- end }}
230246
{{- else }}
231247
{{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }}
232248
{{- $fullname := include "pulsar.fullname" . -}}
@@ -250,6 +266,10 @@ spec:
250266
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
251267
mountPath: /pulsar/data/bookkeeper/ledgers
252268
{{- end }}
269+
{{- if .Values.bookkeeper.volumes.index.enabled }}
270+
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
271+
mountPath: {{ .Values.bookkeeper.volumes.index.mountPath }}
272+
{{- end }}
253273
{{- end }}
254274
{{- if .Values.bookkeeper.extraVolumeMounts }}
255275
{{ toYaml .Values.bookkeeper.extraVolumeMounts | indent 8 }}
@@ -261,6 +281,10 @@ spec:
261281
emptyDir: {}
262282
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}"
263283
emptyDir: {}
284+
{{- if .Values.bookkeeper.volumes.index.enabled }}
285+
- name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
286+
emptyDir: {}
287+
{{- end }}
264288
{{- end }}
265289
{{- include "pulsar.bookkeeper.certs.volumes" . | nindent 6 }}
266290
{{- if .Values.bookkeeper.extraVolumes }}
@@ -288,6 +312,26 @@ spec:
288312
selector:
289313
{{- toYaml . | nindent 10 }}
290314
{{- end }}
315+
{{- if .Values.bookkeeper.volumes.index.enabled }}
316+
- metadata:
317+
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
318+
spec:
319+
accessModes: [ "ReadWriteOnce" ]
320+
resources:
321+
requests:
322+
storage: {{ .Values.bookkeeper.volumes.index.size }}
323+
{{- if .Values.bookkeeper.volumes.index.storageClassName }}
324+
storageClassName: "{{ .Values.bookkeeper.volumes.index.storageClassName }}"
325+
{{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage)) .Values.bookkeeper.volumes.index.storageClass }}
326+
storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
327+
{{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage }}
328+
storageClassName: "local-storage"
329+
{{- end }}
330+
{{- with .Values.bookkeeper.volumes.index.selector }}
331+
selector:
332+
{{- toYaml . | nindent 10 }}
333+
{{- end }}
334+
{{- end }}
291335
{{- else }}
292336
{{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }}
293337
{{- $fullname := include "pulsar.fullname" . -}}
@@ -367,6 +411,26 @@ spec:
367411
{{- toYaml . | nindent 8 }}
368412
{{- end }}
369413
{{- end }}
414+
{{- if .Values.bookkeeper.volumes.index.enabled }}
415+
- metadata:
416+
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
417+
spec:
418+
accessModes: [ "ReadWriteOnce" ]
419+
resources:
420+
requests:
421+
storage: {{ .Values.bookkeeper.volumes.index.size }}
422+
{{- if .Values.bookkeeper.volumes.index.storageClassName }}
423+
storageClassName: "{{ .Values.bookkeeper.volumes.index.storageClassName }}"
424+
{{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage)) .Values.bookkeeper.volumes.index.storageClass }}
425+
storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
426+
{{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage }}
427+
storageClassName: "local-storage"
428+
{{- end }}
429+
{{- with .Values.bookkeeper.volumes.index.selector }}
430+
selector:
431+
{{- toYaml . | nindent 8 }}
432+
{{- end }}
433+
{{- end }}
370434
{{- end }}
371435
{{- end }}
372436
{{- end }}

‎charts/pulsar/templates/bookkeeper-storageclass.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,21 @@ parameters:
3636
type: {{ .Values.bookkeeper.volumes.common.storageClass.type }}
3737
fsType: {{ .Values.bookkeeper.volumes.common.storageClass.fsType }}
3838
{{- end }}
39+
{{- if and .Values.bookkeeper.volumes.index.enabled (not .Values.bookkeeper.volumes.index.local_storage) .Values.bookkeeper.volumes.index.storageClass }}
40+
---
41+
apiVersion: storage.k8s.io/v1
42+
kind: StorageClass
43+
metadata:
44+
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
45+
namespace: {{ template "pulsar.namespace" . }}
46+
labels:
47+
{{- include "pulsar.standardLabels" . | nindent 4 }}
48+
component: {{ .Values.bookkeeper.component }}
49+
provisioner: {{ .Values.bookkeeper.volumes.index.storageClass.provisioner }}
50+
parameters:
51+
type: {{ .Values.bookkeeper.volumes.index.storageClass.type }}
52+
fsType: {{ .Values.bookkeeper.volumes.index.storageClass.fsType }}
53+
{{- end }}
3954
{{- else }}
4055

4156
{{- if and (not .Values.bookkeeper.volumes.journal.local_storage) .Values.bookkeeper.volumes.journal.storageClass }}
@@ -67,6 +82,21 @@ parameters:
6782
type: {{ .Values.bookkeeper.volumes.ledgers.storageClass.type }}
6883
fsType: {{ .Values.bookkeeper.volumes.ledgers.storageClass.fsType }}
6984
{{- end }}
85+
{{- if and .Values.bookkeeper.volumes.index.enabled (not .Values.bookkeeper.volumes.index.local_storage) .Values.bookkeeper.volumes.index.storageClass }}
86+
---
87+
apiVersion: storage.k8s.io/v1
88+
kind: StorageClass
89+
metadata:
90+
name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}"
91+
namespace: {{ template "pulsar.namespace" . }}
92+
labels:
93+
{{- include "pulsar.standardLabels" . | nindent 4 }}
94+
component: {{ .Values.bookkeeper.component }}
95+
provisioner: {{ .Values.bookkeeper.volumes.index.storageClass.provisioner }}
96+
parameters:
97+
type: {{ .Values.bookkeeper.volumes.index.storageClass.type }}
98+
fsType: {{ .Values.bookkeeper.volumes.index.storageClass.fsType }}
99+
{{- end }}
70100
{{- end }}
71101

72102
{{- end }}

‎charts/pulsar/values.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -818,6 +818,19 @@ bookkeeper:
818818
size: 10Gi
819819
# storageClassName: existent-storage-class
820820
mountPath: /pulsar/data/bookkeeper/ledgers1
821+
# Optional dedicated volume for BookKeeper indexes / RocksDB (`indexDirectories`).
822+
# Useful when ledgers are on HDD but you want indexes on NVMe.
823+
index:
824+
enabled: true
825+
name: index
826+
size: 10Gi
827+
local_storage: true
828+
storageClassName: existent-storage-class
829+
# storageClass:
830+
# ...
831+
# selector:
832+
# ...
833+
mountPath: /pulsar/data/bookkeeper/index
821834

822835
## use a single common volume for both journal and ledgers
823836
useSingleCommonVolume: false
@@ -830,6 +843,14 @@ bookkeeper:
830843
# ...
831844
# selector:
832845
# ...
846+
847+
# Configure BookKeeper `indexDirectories` (RocksDB/index location for DbLedgerStorage).
848+
# When empty/unset, BookKeeper falls back to `ledgerDirectories`.
849+
# If `bookkeeper.volumes.index.enabled=true` and this is empty, it defaults to `bookkeeper.volumes.index.mountPath`.
850+
# Example: place indexes on NVMe by reusing the journal volume mount:
851+
# indexDirectories:
852+
# - /pulsar/data/bookkeeper/journal/index
853+
indexDirectories:
833854
## Bookkeeper service account
834855
## templates/bookkeeper-service-account.yaml
835856
service_account:

0 commit comments

Comments
 (0)