Add workers.celery.hpa

chart/newsfragments/64734.significant.rst

@@ -0,0 +1 @@
+``workers.hpa`` section is now deprecated in favor of ``workers.celery.hpa``. Please update your configuration accordingly.

chart/templates/NOTES.txt

@@ -517,6 +517,46 @@ DEPRECATION WARNING:
 
 {{- end }}
 
+{{- if .Values.workers.hpa.enabled }}
+
+ DEPRECATION WARNING:
+    `workers.hpa.enabled` has been renamed to `workers.celery.hpa.enabled`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.hpa.minReplicaCount) 0 }}
+
+ DEPRECATION WARNING:
+    `workers.hpa.minReplicaCount` has been renamed to `workers.celery.hpa.minReplicaCount`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.hpa.maxReplicaCount) 5 }}
+
+ DEPRECATION WARNING:
+    `workers.hpa.maxReplicaCount` has been renamed to `workers.celery.hpa.maxReplicaCount`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if ne (toJson .Values.workers.hpa.metrics | quote) (toJson "[{\"resource\":{\"name\":\"cpu\",\"target\":{\"averageUtilization\":80,\"type\":\"Utilization\"}},\"type\":\"Resource\"}]") }}
+
+ DEPRECATION WARNING:
+    `workers.hpa.metrics` has been renamed to `workers.celery.hpa.metrics`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.hpa.behavior) }}
+
+ DEPRECATION WARNING:
+    `workers.hpa.behavior` has been renamed to `workers.celery.hpa.behavior`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
 {{- if not .Values.workers.persistence.enabled }}
 
  DEPRECATION WARNING:

chart/templates/workers/worker-deployment.yaml

@@ -32,11 +32,11 @@
   {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}}
   {{- $_ := set $globals.Values "workers" $workers -}}
   {{- with $globals -}}
+{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
+---
 {{- $persistence := or .Values.workers.persistence.enabled }}
 {{- $keda := .Values.workers.keda.enabled }}
 {{- $hpa := and .Values.workers.hpa.enabled (not .Values.workers.keda.enabled) }}
-{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }}
----
 {{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }}
 {{- $affinity := or .Values.workers.affinity .Values.affinity }}
 {{- $tolerations := or .Values.workers.tolerations .Values.tolerations }}

chart/values.schema.json

@@ -1951,27 +1951,27 @@
                     }
                 },
                 "hpa": {
-                    "description": "HPA configuration for Airflow Celery workers.",
+                    "description": "HPA configuration for Airflow Celery workers (deprecated, use ``workers.celery.hpa`` instead).",
                     "type": "object",
                     "additionalProperties": false,
                     "properties": {
                         "enabled": {
-                            "description": "Allow HPA autoscaling (KEDA must be disabled).",
+                            "description": "Allow HPA autoscaling (KEDA must be disabled) (deprecated, use ``workers.celery.hpa.enabled`` instead).",
                             "type": "boolean",
                             "default": false
                         },
                         "minReplicaCount": {
-                            "description": "Minimum number of Airflow Celery workers created by HPA.",
+                            "description": "Minimum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.minReplicaCount`` instead).",
                             "type": "integer",
                             "default": 0
                         },
                         "maxReplicaCount": {
-                            "description": "Maximum number of Airflow Celery workers created by HPA.",
+                            "description": "Maximum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.maxReplicaCount`` instead).",
                             "type": "integer",
                             "default": 5
                         },
                         "metrics": {
-                            "description": "Specifications for which to use to calculate the desired replica count.",
+                            "description": "Specifications for which to use to calculate the desired replica count (deprecated, use ``workers.celery.hpa.metrics`` instead).",
                             "type": "array",
                             "default": [
                                 {
@@ -1990,7 +1990,7 @@
                             }
                         },
                         "behavior": {
-                            "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.",
+                            "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use ``workers.celery.hpa.behavior`` instead).",
                             "type": "object",
                             "default": {},
                             "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior"
@@ -3051,6 +3051,54 @@
                                 }
                             }
                         },
+                        "hpa": {
+                            "description": "HPA configuration for Airflow Celery workers.",
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "enabled": {
+                                    "description": "Allow HPA autoscaling (KEDA must be disabled).",
+                                    "type": [
+                                        "boolean",
+                                        "null"
+                                    ],
+                                    "default": null
+                                },
+                                "minReplicaCount": {
+                                    "description": "Minimum number of Airflow Celery workers created by HPA.",
+                                    "type": [
+                                        "integer",
+                                        "null"
+                                    ],
+                                    "default": null
+                                },
+                                "maxReplicaCount": {
+                                    "description": "Maximum number of Airflow Celery workers created by HPA.",
+                                    "type": [
+                                        "integer",
+                                        "null"
+                                    ],
+                                    "default": null
+                                },
+                                "metrics": {
+                                    "description": "Specifications for which to use to calculate the desired replica count.",
+                                    "type": [
+                                        "array",
+                                        "null"
+                                    ],
+                                    "default": null,
+                                    "items": {
+                                        "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec"
+                                    }
+                                },
+                                "behavior": {
+                                    "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.",
+                                    "type": "object",
+                                    "default": {},
+                                    "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior"
+                                }
+                            }
+                        },
                         "persistence": {
                             "description": "Persistence configuration for Airflow Celery workers.",
                             "type": "object",

chart/values.yaml

@@ -865,16 +865,21 @@ workers:
     usePgbouncer: true
 
   # Allow HPA for Airflow Celery workers (KEDA must be disabled)
+  # (deprecated, use `workers.celery.hpa` instead)
   hpa:
+    # (deprecated, use `workers.celery.hpa.enabled` instead)
     enabled: false
 
     # Minimum number of Airflow Celery workers created by HPA
+    # (deprecated, use `workers.celery.hpa.minReplicaCount` instead)
     minReplicaCount: 0
 
     # Maximum number of Airflow Celery workers created by HPA
+    # (deprecated, use `workers.celery.hpa.maxReplicaCount` instead)
     maxReplicaCount: 5
 
     # Specifications for which to use to calculate the desired replica count
+    # (deprecated, use `workers.celery.hpa.metrics` instead)
     metrics:
       - type: Resource
         resource:
@@ -884,6 +889,7 @@ workers:
             averageUtilization: 80
 
     # Scaling behavior of the target in both Up and Down directions
+    # (deprecated, use `workers.celery.hpa.behavior` instead)
     behavior: {}
 
   # Persistence volume configuration for Airflow Celery workers
@@ -1334,6 +1340,22 @@ workers:
       # This configuration will be ignored if PGBouncer is not enabled
       usePgbouncer: ~
 
+    # Allow HPA for Airflow Celery workers (KEDA must be disabled)
+    hpa:
+      enabled: ~
+
+      # Minimum number of Airflow Celery workers created by HPA
+      minReplicaCount: ~
+
+      # Maximum number of Airflow Celery workers created by HPA
+      maxReplicaCount: ~
+
+      # Specifications for which to use to calculate the desired replica count
+      metrics: ~
+
+      # Scaling behavior of the target in both Up and Down directions
+      behavior: {}
+
     # Persistence volume configuration for Airflow Celery workers
     persistence:
       # Enable persistent volumes