From c74b4db8b4a50c927e5d7c1e4fba7fa63aee3b32 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Sat, 27 Dec 2025 20:14:24 +0800 Subject: [PATCH 1/4] Add support for BookKeeper indexDirectories configuration --- charts/pulsar/templates/_bookkeeper.tpl | 19 ++++++ .../templates/bookkeeper-configmap.yaml | 6 ++ .../templates/bookkeeper-statefulset.yaml | 64 +++++++++++++++++++ .../templates/bookkeeper-storageclass.yaml | 30 +++++++++ charts/pulsar/values.yaml | 21 ++++++ 5 files changed, 140 insertions(+) diff --git a/charts/pulsar/templates/_bookkeeper.tpl b/charts/pulsar/templates/_bookkeeper.tpl index c8c97d0b..610d8030 100644 --- a/charts/pulsar/templates/_bookkeeper.tpl +++ b/charts/pulsar/templates/_bookkeeper.tpl @@ -170,6 +170,25 @@ PULSAR_PREFIX_tlsTrustStore: {{ ternary "/pulsar/certs/cacerts/ca-combined.pem" {{- end }} {{- end }} +{{/* +Render BookKeeper indexDirectories as comma-separated string. +Accepts either a string or a list of strings. +*/}} +{{- define "pulsar.bookkeeper.indexDirectories" -}} +{{- $v := .Values.bookkeeper.indexDirectories -}} +{{- if $v -}} +{{- if kindIs "string" $v -}} +{{- $v -}} +{{- else -}} +{{- $v | join "," -}} +{{- end -}} +{{- else if and .Values.bookkeeper.volumes.index.enabled .Values.bookkeeper.volumes.index.mountPath -}} +{{- .Values.bookkeeper.volumes.index.mountPath -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + {{/* Define bookie init container : verify cluster id */}} diff --git a/charts/pulsar/templates/bookkeeper-configmap.yaml b/charts/pulsar/templates/bookkeeper-configmap.yaml index f133f1f4..138e1020 100644 --- a/charts/pulsar/templates/bookkeeper-configmap.yaml +++ b/charts/pulsar/templates/bookkeeper-configmap.yaml @@ -55,6 +55,12 @@ data: {{- else }} ledgerDirectories: "/pulsar/data/bookkeeper/ledgers" {{- end }} + {{- $indexDirs := include "pulsar.bookkeeper.indexDirectories" . }} + {{- if $indexDirs }} + # Configure RocksDB/index location for DbLedgerStorage. When unset, BookKeeper falls back to ledgerDirectories. + indexDirectories: {{ $indexDirs | quote }} + PULSAR_PREFIX_indexDirectories: {{ $indexDirs | quote }} + {{- end }} {{- if .Values.functions.useBookieAsStateStore }} # Stateful function config extraServerComponents: "org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent" diff --git a/charts/pulsar/templates/bookkeeper-statefulset.yaml b/charts/pulsar/templates/bookkeeper-statefulset.yaml index 5bd5fa4b..414fabda 100644 --- a/charts/pulsar/templates/bookkeeper-statefulset.yaml +++ b/charts/pulsar/templates/bookkeeper-statefulset.yaml @@ -212,6 +212,18 @@ spec: nohup /pulsar/bin/certs-combine-pem-infinity.sh /pulsar/certs/cacerts/ca-combined.pem {{ template "pulsar.certs.cacerts" (dict "certs" .Values.tls.bookie.cacerts.certs) }} > /pulsar/certs/cacerts/certs-combine-pem-infinity.log 2>&1 & cd /pulsar; {{- end }} + {{- $indexDirs := include "pulsar.bookkeeper.indexDirectories" . -}} + {{- if $indexDirs }} + INDEX_DIRS={{ $indexDirs | quote }} + if [ -n "$INDEX_DIRS" ]; then + oldIFS=$IFS + IFS=',' + for dir in $INDEX_DIRS; do + mkdir -p "$dir" + done + IFS=$oldIFS + fi + {{- end }} bin/apply-config-from-env.py conf/bookkeeper.conf; {{- include "pulsar.bookkeeper.zookeeper.tls.settings" . | nindent 10 }} OPTS="${OPTS} -Dlog4j2.formatMsgNoLookups=true" exec bin/pulsar bookie; @@ -227,6 +239,10 @@ spec: {{- if .Values.bookkeeper.volumes.useSingleCommonVolume }} - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.common.name }}" mountPath: /pulsar/data/bookkeeper + {{- if .Values.bookkeeper.volumes.index.enabled }} + - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + mountPath: {{ .Values.bookkeeper.volumes.index.mountPath }} + {{- end }} {{- else }} {{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }} {{- $fullname := include "pulsar.fullname" . -}} @@ -250,6 +266,10 @@ spec: - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}" mountPath: /pulsar/data/bookkeeper/ledgers {{- end }} + {{- if .Values.bookkeeper.volumes.index.enabled }} + - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + mountPath: {{ .Values.bookkeeper.volumes.index.mountPath }} + {{- end }} {{- end }} {{- if .Values.bookkeeper.extraVolumeMounts }} {{ toYaml .Values.bookkeeper.extraVolumeMounts | indent 8 }} @@ -261,6 +281,10 @@ spec: emptyDir: {} - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.ledgers.name }}" emptyDir: {} + {{- if .Values.bookkeeper.volumes.index.enabled }} + - name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + emptyDir: {} + {{- end }} {{- end }} {{- include "pulsar.bookkeeper.certs.volumes" . | nindent 6 }} {{- if .Values.bookkeeper.extraVolumes }} @@ -288,6 +312,26 @@ spec: selector: {{- toYaml . | nindent 10 }} {{- end }} + {{- if .Values.bookkeeper.volumes.index.enabled }} + - metadata: + name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.bookkeeper.volumes.index.size }} + {{- if .Values.bookkeeper.volumes.index.storageClassName }} + storageClassName: "{{ .Values.bookkeeper.volumes.index.storageClassName }}" + {{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage)) .Values.bookkeeper.volumes.index.storageClass }} + storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + {{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage }} + storageClassName: "local-storage" + {{- end }} + {{- with .Values.bookkeeper.volumes.index.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} {{- else }} {{- if .Values.bookkeeper.volumes.journal.useMultiVolumes }} {{- $fullname := include "pulsar.fullname" . -}} @@ -367,6 +411,26 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- end }} + {{- if .Values.bookkeeper.volumes.index.enabled }} + - metadata: + name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.bookkeeper.volumes.index.size }} + {{- if .Values.bookkeeper.volumes.index.storageClassName }} + storageClassName: "{{ .Values.bookkeeper.volumes.index.storageClassName }}" + {{- else if and (not (and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage)) .Values.bookkeeper.volumes.index.storageClass }} + storageClassName: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + {{- else if and .Values.volumes.local_storage .Values.bookkeeper.volumes.index.local_storage }} + storageClassName: "local-storage" + {{- end }} + {{- with .Values.bookkeeper.volumes.index.selector }} + selector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/pulsar/templates/bookkeeper-storageclass.yaml b/charts/pulsar/templates/bookkeeper-storageclass.yaml index a105d89d..2d600701 100644 --- a/charts/pulsar/templates/bookkeeper-storageclass.yaml +++ b/charts/pulsar/templates/bookkeeper-storageclass.yaml @@ -36,6 +36,21 @@ parameters: type: {{ .Values.bookkeeper.volumes.common.storageClass.type }} fsType: {{ .Values.bookkeeper.volumes.common.storageClass.fsType }} {{- end }} +{{- if and .Values.bookkeeper.volumes.index.enabled (not .Values.bookkeeper.volumes.index.local_storage) .Values.bookkeeper.volumes.index.storageClass }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.bookkeeper.component }} +provisioner: {{ .Values.bookkeeper.volumes.index.storageClass.provisioner }} +parameters: + type: {{ .Values.bookkeeper.volumes.index.storageClass.type }} + fsType: {{ .Values.bookkeeper.volumes.index.storageClass.fsType }} +{{- end }} {{- else }} {{- if and (not .Values.bookkeeper.volumes.journal.local_storage) .Values.bookkeeper.volumes.journal.storageClass }} @@ -67,6 +82,21 @@ parameters: type: {{ .Values.bookkeeper.volumes.ledgers.storageClass.type }} fsType: {{ .Values.bookkeeper.volumes.ledgers.storageClass.fsType }} {{- end }} +{{- if and .Values.bookkeeper.volumes.index.enabled (not .Values.bookkeeper.volumes.index.local_storage) .Values.bookkeeper.volumes.index.storageClass }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "{{ template "pulsar.fullname" . }}-{{ .Values.bookkeeper.component }}-{{ .Values.bookkeeper.volumes.index.name }}" + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.bookkeeper.component }} +provisioner: {{ .Values.bookkeeper.volumes.index.storageClass.provisioner }} +parameters: + type: {{ .Values.bookkeeper.volumes.index.storageClass.type }} + fsType: {{ .Values.bookkeeper.volumes.index.storageClass.fsType }} +{{- end }} {{- end }} {{- end }} diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 723a9d9e..11128570 100755 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -818,6 +818,19 @@ bookkeeper: size: 10Gi # storageClassName: existent-storage-class mountPath: /pulsar/data/bookkeeper/ledgers1 + # Optional dedicated volume for BookKeeper indexes / RocksDB (`indexDirectories`). + # Useful when ledgers are on HDD but you want indexes on NVMe. + index: + enabled: true + name: index + size: 10Gi + local_storage: true + storageClassName: existent-storage-class + # storageClass: + # ... + # selector: + # ... + mountPath: /pulsar/data/bookkeeper/index ## use a single common volume for both journal and ledgers useSingleCommonVolume: false @@ -830,6 +843,14 @@ bookkeeper: # ... # selector: # ... + + # Configure BookKeeper `indexDirectories` (RocksDB/index location for DbLedgerStorage). + # When empty/unset, BookKeeper falls back to `ledgerDirectories`. + # If `bookkeeper.volumes.index.enabled=true` and this is empty, it defaults to `bookkeeper.volumes.index.mountPath`. + # Example: place indexes on NVMe by reusing the journal volume mount: + # indexDirectories: + # - /pulsar/data/bookkeeper/journal/index + indexDirectories: ## Bookkeeper service account ## templates/bookkeeper-service-account.yaml service_account: From 38cb958f4843ae37d0cc4124f2ed589b24fa0274 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 5 Jan 2026 19:29:10 +0800 Subject: [PATCH 2/4] Disable BookKeeper indexDirectories by default --- charts/pulsar/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 11128570..05796360 100755 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -821,7 +821,7 @@ bookkeeper: # Optional dedicated volume for BookKeeper indexes / RocksDB (`indexDirectories`). # Useful when ledgers are on HDD but you want indexes on NVMe. index: - enabled: true + enabled: false name: index size: 10Gi local_storage: true From 6f6bf5c0c0f564d736f19ed0bee4759840f4b6de Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 5 Jan 2026 20:44:58 +0800 Subject: [PATCH 3/4] Comment out storageClassName for index configuration in values.yaml --- charts/pulsar/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index 05796360..da051f10 100755 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -825,7 +825,7 @@ bookkeeper: name: index size: 10Gi local_storage: true - storageClassName: existent-storage-class + # storageClassName: existent-storage-class # storageClass: # ... # selector: From 078dac4fd3facc3156fd6cccae6e5fdd55c7c93e Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 5 Jan 2026 21:04:37 +0800 Subject: [PATCH 4/4] Remove PULSAR_PREFIX_indexDirectories from BookKeeper configuration --- charts/pulsar/templates/bookkeeper-configmap.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/pulsar/templates/bookkeeper-configmap.yaml b/charts/pulsar/templates/bookkeeper-configmap.yaml index 138e1020..db5e52fc 100644 --- a/charts/pulsar/templates/bookkeeper-configmap.yaml +++ b/charts/pulsar/templates/bookkeeper-configmap.yaml @@ -59,7 +59,6 @@ data: {{- if $indexDirs }} # Configure RocksDB/index location for DbLedgerStorage. When unset, BookKeeper falls back to ledgerDirectories. indexDirectories: {{ $indexDirs | quote }} - PULSAR_PREFIX_indexDirectories: {{ $indexDirs | quote }} {{- end }} {{- if .Values.functions.useBookieAsStateStore }} # Stateful function config