feat: Replace credentials-secret with db/broker connection(s)

CHANGELOG.md

@@ -14,6 +14,10 @@
 
 - Gracefully shutdown all concurrent tasks by forwarding the SIGTERM signal ([#741]).
 - Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#733]).
+- BREAKING: Replace Airflow credentials-secret with database/broker connections ([#743]).
+  - Existing secret is retained for the airflow admin user alone.
+  - Database/broker connections can be defined either using structs or as a generic connection string (see ADR 29).
+  - Removed standalone examples folder (not affecting the documentation).
 
 ### Fixed
 
@@ -28,6 +32,7 @@
 [#734]: https://github.com/stackabletech/airflow-operator/pull/734
 [#741]: https://github.com/stackabletech/airflow-operator/pull/741
 [#742]: https://github.com/stackabletech/airflow-operator/pull/742
+[#743]: https://github.com/stackabletech/airflow-operator/pull/743
 
 ## [25.11.0] - 2025-11-07
 

docs/modules/airflow/examples/example-airflow-dags-configmap.yaml

@@ -7,9 +7,12 @@ spec:
   image:
     productVersion: 3.1.6
   clusterConfig:
-    loadExamples: false
-    exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: admin-user-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: postgresql-credentials
     volumes:
       - name: cm-dag # <3>
         configMap:
@@ -23,18 +26,8 @@ spec:
       listenerClass: external-unstable
     roleGroups:
       default:
-        envOverrides:
+        envOverrides: &envOverrides
           AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
         replicas: 1
   celeryExecutors:
-    roleGroups:
-      default:
-        envOverrides:
-          AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
-        replicas: 2
-  schedulers:
-    roleGroups:
-      default:
-        envOverrides:
-          AIRFLOW__CORE__DAGS_FOLDER: "/dags" # <8>
-        replicas: 1
+    ...

docs/modules/airflow/examples/example-airflow-incluster.yaml

@@ -9,28 +9,39 @@ spec:
   clusterConfig:
     loadExamples: false
     exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: admin-user-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: postgresql-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
     roleGroups:
       default:
-        envOverrides:
+        envOverrides: &envOverrides
           AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
         replicas: 1
   schedulers:
     roleGroups:
       default:
-        envOverrides:
-          AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+        envOverrides: *envOverrides
         replicas: 1
   celeryExecutors:
+    celeryResultBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: postgresql-credentials
+    celeryBrokerUrl:
+      redis:
+        host: airflow-redis-master
+        credentialsSecret: redis-credentials
     roleGroups:
       default:
-        envOverrides:
-          AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+        envOverrides: *envOverrides
         replicas: 1
 # in case of using kubernetesExecutors
 #  kubernetesExecutors:
-#    envOverrides:
-#      AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
+#    envOverrides: *envOverrides

docs/modules/airflow/examples/getting_started/code/airflow-credentials.yaml

@@ -2,15 +2,27 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: simple-airflow-credentials
+  name: airflow-admin-user-credentials
 type: Opaque
 stringData:
   adminUser.username: airflow
   adminUser.firstname: Airflow
   adminUser.lastname: Admin
   adminUser.email: [email protected]
   adminUser.password: airflow
-  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:[email protected]/airflow
-  # Only needed when using celery workers (instead of Kubernetes executors)
-  connections.celeryResultBackend: db+postgresql://airflow:[email protected]/airflow
-  connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-postgresql-credentials
+stringData:
+  username: airflow
+  password: airflow
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-redis-credentials
+stringData:
+  username: ""
+  password: redis

docs/modules/airflow/examples/getting_started/code/airflow.yaml

@@ -10,14 +10,28 @@ spec:
   clusterConfig:
     loadExamples: true
     exposeConfig: false
-    credentialsSecret: simple-airflow-credentials
+    credentialsSecret: airflow-admin-user-credentials
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1
   celeryExecutors:
+    celeryResultBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: airflow-postgresql-credentials
+    celeryBrokerUrl:
+      redis:
+        host: airflow-redis-master
+        credentialsSecret: airflow-redis-credentials
     roleGroups:
       default:
         replicas: 1

docs/modules/airflow/pages/getting_started/first_steps.adoc

@@ -10,9 +10,10 @@ With the external dependencies required by Airflow (Postgresql and Redis) instal
 
 Supported versions for PostgreSQL and Redis can be found in the https://airflow.apache.org/docs/apache-airflow/stable/installation/prerequisites.html#prerequisites[Airflow documentation].
 
-== Secret with Airflow credentials
+== Airflow secrets
 
-Create a Secret with the necessary credentials, this entails database connection credentials as well as an admin account for Airflow itself.
+Secrets are required for the mandatory metadata database connection and the Airflow admin user.
+When using the celery executor it is also required to provide information for the celery database and broker.
 Create a file called `airflow-credentials.yaml`:
 
 [source,yaml]
@@ -23,13 +24,12 @@ And apply it:
 [source,bash]
 include::example$getting_started/code/getting_started.sh[tag=apply-airflow-credentials]
 
-`connections.sqlalchemyDatabaseUri` must contain the connection string to the SQL database storing the Airflow metadata.
+`airflow-postgresql-credentials` contains credentials for the SQL database storing the Airflow metadata.
+In this example we will use the same database for both the Airflow job metadata as well as the Celery broker metadata.
 
-`connections.celeryResultBackend` must contain the connection string to the SQL database storing the job metadata (the example above uses the same PostgreSQL database for both).
+`airflow-redis-credentials` contains credentials for the the Redis instance used for queuing the jobs submitted to the Airflow executor(s).
 
-`connections.celeryBrokerUrl` must contain the connection string to the Redis instance used for queuing the jobs submitted to the airflow executor(s).
-
-The `adminUser` fields are used to create an admin user.
+`airflow-admin-user-credentials`: the `adminUser.*` fields are used to create the initial admin user.
 
 NOTE: The admin user is disabled if you use a non-default authentication mechanism like LDAP.
 
@@ -60,21 +60,27 @@ include::example$getting_started/code/getting_started.sh[tag=install-airflow]
 
 Where:
 
-* `metadata.name` contains the name of the Airflow cluster.
+* `metadata.name`: contains the name of the Airflow cluster.
+* `spec.clusterConfig.metadataDatabase`: specifies one of the supported database types (in this case, `postgresql`) along with references to the host, database and the secret containing the connection credentials.
 * the product version of the Docker image provided by Stackable must be set in `spec.image.productVersion`.
 * `spec.celeryExecutors`: deploy executors managed by Airflow's Celery engine.
 Alternatively you can use `kuberenetesExectors` that use Airflow's Kubernetes engine for executor management.
 For more information see https://airflow.apache.org/docs/apache-airflow/stable/executor/index.html#executor-types).
-* the `spec.clusterConfig.loadExamples` key is optional and defaults to `false`.
+* `spec.celeryExecutors.celeryResultBackend`: specifies one of the supported database types (in this case, `postgresql`) along with references to the host, database and the secret containing the connection credentials.
+* `spec.celeryExecutors.celeryBrokerUrl`: specifies one of the supported queue/broker types (in this case, `redis`) along with references to the host and the secret containing the connection credentials.
+* `spec.clusterConfig.loadExamples`: this key is optional and defaults to `false`.
 It is set to `true` here as the example DAGs are used when verifying the installation.
-* the `spec.clusterConfig.exposeConfig` key is optional and defaults to `false`. It is set to `true` only as an aid to verify the configuration and should never be used as such in anything other than test or demo clusters.
-* the previously created secret must be referenced in `spec.clusterConfig.credentialsSecret`.
+* `spec.clusterConfig.exposeConfig`: this key is optional and defaults to `false`.
+It is set to `true` only as an aid to verify the configuration and should never be used as such in anything other than test or demo clusters.
+* `spec.clusterConfig.credentialsSecret`: specifies the secret containing the Airflow admin user information.
 
 NOTE: The version you need to specify for `spec.image.productVersion` is the desired version of Apache Airflow.
 You can optionally specify the `spec.image.stackableVersion` to a certain release like `23.11.0` but it is recommended to leave it out and use the default provided by the operator.
 Check our https://oci.stackable.tech/[image registry,window=_blank] for a list of available versions. Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank].
 It should generally be safe to simply use the latest version that is available.
 
+NOTE: Refer to xref:usage-guide/db-connect.adoc[] for more information about database/broker connections.
+
 This creates the actual Airflow cluster.
 
 After a while, all the Pods in the StatefulSets should be ready:

docs/modules/airflow/pages/usage-guide/db-connect.adoc

@@ -0,0 +1,77 @@
+= Database connections
+:description: Configure Airflow Database connectivity.
+
+Airflow requires a metadata database for storing e.g. DAG, task and job data.
+The actual connection string is provided by the operator so that the user does not need to remember the exact structure.
+The same database can be accessed using different drivers (e.g. job metadata vs. queued job metadata).
+
+== Typed connections
+
+[source,yaml]
+----
+spec:
+  clusterConfig:
+    metadataDatabase:
+      postgresql: # <1>
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: postgresql-credentials # <2>
+----
+<1> A reference to one of the supported database backends (e.g. `postgresql`).
+<2> A reference to a Secret which must contain the two fields `username` and `password`.
+
+The queue/broker metadata and URL is only needed when running the celery executor.
+The `celeryResultBackend` definition uses the same structure as `metadataDatabase` shown above.
+The `celeryBrokerUrl` definition is similar but does not require a `databaseName`.
+
+[source,yaml]
+----
+---
+spec:
+  celeryExecutors:
+    celeryResultBackend:
+      postgresql: # <1>
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecret: postgresql-credentials # <2>
+    celeryBrokerUrl:
+      redis: # <3>
+        host: airflow-redis-master
+        credentialsSecret: redis-credentials # <2>
+----
+<1> A reference to one of the supported database backends (e.g. `postgresql`).
+<2> A reference to a secret which must contain the two fields `username` and `password`.
+<3> A reference to one of the supported queue brokers (e.g. `redis`).
+
+== Generic connections
+
+Alternatively, these connections can also be defined in full in a referenced secret:
+
+[source,yaml]
+----
+---
+spec:
+  clusterConfig:
+    metadataDatabase:
+      generic:
+        uriSecret: postgresql-metadata # <1>
+----
+
+[source,yaml]
+----
+---
+spec:
+  celeryResultBackend:
+      generic:
+        uriSecret: postgresql-celery # <2>
+    celeryBrokerUrl:
+      generic:
+        uriSecret: redis-celery # <3>
+----
+
+<1> A reference to a secret which must contain the single fields `uri` e.g.
+`uri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow`
+<2> A reference to a secret which must contain the single fields `uri` e.g.
+`uri: db+postgresql://airflow:airflow@airflow-postgresql/airflow`
+<3> A reference to a secret which must contain the single fields `uri` e.g.
+`uri: redis://:redis@airflow-redis-master:6379/0`

docs/modules/airflow/pages/usage-guide/logging.adoc

@@ -23,6 +23,7 @@ spec:
               "flask_appbuilder":
                 level: WARN
   celeryExecutors:
+    ...
     config:
       logging:
         enableVectorAgent: true

docs/modules/airflow/pages/usage-guide/storage-resources.adoc

@@ -27,6 +27,7 @@ spec:
       default:
         replicas: 2
   celeryExecutors:
+    ...
     config:
       resources:
         cpu:

docs/modules/airflow/pages/usage-guide/using-kubernetes-executors.adoc

@@ -12,6 +12,7 @@ E.g. you would change the following example
 ----
 spec:
   celeryExecutors:
+    ...
     roleGroups:
       default:
         replicas: 2

docs/modules/airflow/partials/nav.adoc

@@ -4,6 +4,7 @@
 * xref:airflow:required-external-components.adoc[]
 * xref:airflow:usage-guide/index.adoc[]
 ** xref:airflow:usage-guide/db-init.adoc[]
+** xref:airflow:usage-guide/db-connect.adoc[]
 ** xref:airflow:usage-guide/mounting-dags.adoc[]
 ** xref:airflow:usage-guide/applying-custom-resources.adoc[]
 ** xref:airflow:usage-guide/listenerclass.adoc[]