Skip to content

Add configuration for graceful shutdown timeout #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
GrigorievNick wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Add configuration for graceful shutdown timeout #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
278616b
Add configuration for graceful shutdown timeout
GrigorievNick Jan 18, 2023
7535ee8
fix templates
GrigorievNick Jan 19, 2023
3ced8a9
fix templates
GrigorievNick Jan 19, 2023
bfa7bed
fix templates
GrigorievNick Jan 19, 2023
44d455d
fix templates
GrigorievNick Jan 19, 2023
e5f8bf4
fix templates
GrigorievNick Jan 19, 2023
3f4a916
fix templates
GrigorievNick Jan 19, 2023
96ca5ec
test submission web hook
GrigorievNick Jan 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test-charts.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
fi

- name: Run chart-testing (lint)
run: ct lint --target-branch ${{ github.event.repository.default_branch }}
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --check-version-increment=false --validate-yaml=false

- name: Create kind cluster
uses: helm/kind-action@v1.4.0
Expand Down
13 changes: 13 additions & 0 deletions charts/dremio_v2/templates/_helpers_coordinator.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,4 +150,17 @@ Coordinator - Pod Tolerations
tolerations:
{{- toYaml $coordinatorTolerations | nindent 2 }}
{{- end -}}
{{- end -}}

{{/*
Coordinator - grace termination period Resource Request
*/}}
{{- define "dremio.coordinator.stopTimeoutPod" -}}
{{- $stopTimeout := $.Values.coordinator.stopTimeout -}}
{{- $stopTimeout -}}
{{- end -}}

{{- define "dremio.coordinator.stopTimeoutDremio" -}}
{{- $stopTimeout := $.Values.coordinator.stopTimeout -}}
{{- sub $stopTimeout 10 -}}
{{- end -}}
19 changes: 19 additions & 0 deletions charts/dremio_v2/templates/_helpers_executor.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,4 +343,23 @@ Executor - Pod Tolerations
tolerations:
{{- toYaml $engineTolerations | nindent 2 }}
{{- end -}}
{{- end -}}

{{/*
Executor - grace termination period Resource Request
*/}}
{{- define "dremio.executor.stopTimeoutPod" -}}
{{- $context := index . 0 -}}
{{- $engineName := index . 1 -}}
{{- $engineConfiguration := default (dict) (get (default (dict) $context.Values.executor.engineOverride) $engineName) -}}
{{- $engineStopTimeout := default ($context.Values.executor.stopTimeout) $engineConfiguration.stopTimeout -}}
{{- $engineStopTimeout -}}
{{- end -}}

{{- define "dremio.executor.stopTimeoutDremio" -}}
{{- $context := index . 0 -}}
{{- $engineName := index . 1 -}}
{{- $engineConfiguration := default (dict) (get (default (dict) $context.Values.executor.engineOverride) $engineName) -}}
{{- $engineStopTimeout := default ($context.Values.executor.stopTimeout) $engineConfiguration.stopTimeout -}}
{{- sub $engineStopTimeout 10 -}}
{{- end -}}
10 changes: 9 additions & 1 deletion charts/dremio_v2/templates/dremio-coordinator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ spec:
{{- include "dremio.coordinator.podAnnotations" $ | nindent 8 }}
spec:
{{- include "dremio.coordinator.serviceAccount" $ | nindent 6 }}
terminationGracePeriodSeconds: 120
terminationGracePeriodSeconds: {{ template "dremio.coordinator.stopTimeoutPod" }}
{{- include "dremio.coordinator.nodeSelector" $ | nindent 6 }}
{{- include "dremio.coordinator.tolerations" $ | nindent 6 }}
containers:
Expand Down Expand Up @@ -58,6 +58,8 @@ spec:
fieldRef:
fieldPath: status.podIP
{{- end }}
- name: DREMIO_STOP_TIMEOUT
value: "{{ template "dremio.coordinator.stopTimeoutDremio" $ }}"
- name: DREMIO_MAX_HEAP_MEMORY_SIZE_MB
value: "{{ template "dremio.coordinator.heapMemory" $ }}"
- name: DREMIO_MAX_DIRECT_MEMORY_SIZE_MB
Expand All @@ -84,9 +86,15 @@ spec:
value: "/opt/dremio/aws/credentials"
- name: AWS_SHARED_CREDENTIALS_FILE
value: "/opt/dremio/aws/credentials"
lifecycle:
preStop:
exec:
command: [ "/opt/dremio/bin/dremio stop" ]
command: ["/opt/dremio/bin/dremio"]
args: ["start-fg"]
ports:
- containerPort: 9047
name: web
- containerPort: 31010
name: client
- containerPort: 32010
Expand Down
8 changes: 7 additions & 1 deletion charts/dremio_v2/templates/dremio-executor.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
{{- include "dremio.executor.podAnnotations" (list $ $engineName) | nindent 8}}
spec:
{{- include "dremio.executor.serviceAccount" (list $ $engineName) | nindent 6}}
terminationGracePeriodSeconds: 120
terminationGracePeriodSeconds: {{ template "dremio.executor.stopTimeoutPod" (list $ $engineName) }}
{{- include "dremio.executor.nodeSelector" (list $ $engineName) | nindent 6}}
{{- include "dremio.executor.tolerations" (list $ $engineName) | nindent 6}}
containers:
Expand Down Expand Up @@ -60,6 +60,8 @@ spec:
fieldRef:
fieldPath: status.podIP
{{- end }}
- name: DREMIO_STOP_TIMEOUT
value: "{{ template "dremio.executor.stopTimeoutDremio" (list $ $engineName) }}"
- name: DREMIO_MAX_HEAP_MEMORY_SIZE_MB
value: "{{ template "dremio.executor.heapMemory" (list $ $engineName) }}"
- name: DREMIO_MAX_DIRECT_MEMORY_SIZE_MB
Expand Down Expand Up @@ -87,6 +89,10 @@ spec:
value: "/opt/dremio/aws/credentials"
- name: AWS_SHARED_CREDENTIALS_FILE
value: "/opt/dremio/aws/credentials"
lifecycle:
preStop:
exec:
command: [ "/opt/dremio/bin/dremio stop" ]
command: ["/opt/dremio/bin/dremio"]
args: ["start-fg"]
ports:
Expand Down
8 changes: 7 additions & 1 deletion charts/dremio_v2/templates/dremio-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ spec:
values:
- dremio-coordinator
topologyKey: "kubernetes.io/hostname"
terminationGracePeriodSeconds: 120
terminationGracePeriodSeconds: {{ template "dremio.coordinator.stopTimeoutPod" }}
{{- include "dremio.coordinator.nodeSelector" $ | nindent 6 }}
{{- include "dremio.coordinator.tolerations" $ | nindent 6 }}
containers:
Expand Down Expand Up @@ -69,6 +69,8 @@ spec:
fieldRef:
fieldPath: status.podIP
{{- end }}
- name: DREMIO_STOP_TIMEOUT
value: "{{ template "dremio.coordinator.stopTimeoutDremio" $ }}"
- name: DREMIO_MAX_HEAP_MEMORY_SIZE_MB
value: "{{ template "dremio.coordinator.heapMemory" $ }}"
- name: DREMIO_MAX_DIRECT_MEMORY_SIZE_MB
Expand All @@ -95,6 +97,10 @@ spec:
value: "/opt/dremio/aws/credentials"
- name: AWS_SHARED_CREDENTIALS_FILE
value: "/opt/dremio/aws/credentials"
lifecycle:
preStop:
exec:
command: [ "/opt/dremio/bin/dremio stop" ]
command: ["/opt/dremio/bin/dremio"]
args:
- "start-fg"
Expand Down
6 changes: 6 additions & 0 deletions charts/dremio_v2/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ coordinator:
# kubectl create secret tls ${TLS_SECRET_NAME} --key ${KEY_FILE} --cert ${CERT_FILE}
secret: dremio-tls-secret-flight

# seconds
stopTimeout: 120

# Dremio Executor
executor:
# CPU & Memory
Expand Down Expand Up @@ -203,6 +206,9 @@ executor:
#nodeSelector: {}
#tolerations: []

# seconds
stopTimeout: 120

# Engine Overrides
#
# The settings above are overridable on a per-engine basis. These
Expand Down