diff --git a/antora.yml b/antora.yml index 7cdcd7c..03ffc62 100644 --- a/antora.yml +++ b/antora.yml @@ -4,12 +4,7 @@ title: 'Kubernetes Autoscaling for Apache Pulsar' start_page: 'index.adoc' nav: -- modules/getting-started/nav.adoc -- modules/migration/nav.adoc -- modules/authentication/nav.adoc -- modules/scaling-components/nav.adoc -- modules/resource-sets/nav.adoc -- modules/crd-spec/nav.adoc +- modules/ROOT/nav.adoc asciidoc: attributes: diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc new file mode 100644 index 0000000..66ae016 --- /dev/null +++ b/modules/ROOT/nav.adoc @@ -0,0 +1,37 @@ +.Get started +* xref:ROOT:index.adoc[] +* xref:getting-started:index.adoc[] +* xref:getting-started:operator.adoc[] +* xref:getting-started:stack.adoc[] +* xref:getting-started:upgrades.adoc[] + +.Migrate +* xref:migration:migrate-cluster.adoc[] + +.Authentication +* xref:authentication:index.adoc[] +* xref:authentication:auth-jwt.adoc[] +* xref:authentication:auth-tls.adoc[] + +.Scaling components +* xref:scaling-components:index.adoc[] +* xref:scaling-components:autoscale-bookies.adoc[] +* xref:scaling-components:autoscale-brokers.adoc[] +* xref:scaling-components:kafka.adoc[] + +.Resource sets +* xref:resource-sets:index.adoc[] +* xref:resource-sets:bookies.adoc[] +* xref:resource-sets:pods.adoc[] +* xref:resource-sets:proxies.adoc[] +* xref:resource-sets:racks.adoc[] + +.CRD spec +* xref:crd-spec:Autorecovery.openapi.adoc[] +* xref:crd-spec:Bastion.openapi.adoc[] +* xref:crd-spec:BookKeeper.openapi.adoc[] +* xref:crd-spec:Broker.openapi.adoc[] +* xref:crd-spec:FunctionsWorker.openapi.adoc[] +* xref:crd-spec:Proxy.openapi.adoc[] +* xref:crd-spec:PulsarCluster.openapi.adoc[] +* xref:crd-spec:ZooKeeper.openapi.adoc[] \ No newline at end of file diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index a95fc46..ac75758 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -1,4 +1,5 @@ = {pulsar-operator-full-name} +:navtitle: About {pulsar-operator-full-name} {pulsar-operator-full-name} simplifies running https://pulsar.apache.org[Apache Pulsar] on Kubernetes by applying the familiar https://kubernetes.io/docs/concepts/extend-kubernetes/operator/[Operator pattern] to Pulsar's components, and horizonally scaling resources up or down based on CPU and memory workloads. diff --git a/modules/authentication/nav.adoc b/modules/authentication/nav.adoc deleted file mode 100644 index c2cc2a2..0000000 --- a/modules/authentication/nav.adoc +++ /dev/null @@ -1,4 +0,0 @@ -.Authentication -* xref:index.adoc[] -* xref:auth-jwt.adoc[] -* xref:auth-tls.adoc[] diff --git a/modules/authentication/pages/auth-jwt.adoc b/modules/authentication/pages/auth-jwt.adoc index 7240021..729610c 100644 --- a/modules/authentication/pages/auth-jwt.adoc +++ b/modules/authentication/pages/auth-jwt.adoc @@ -1,6 +1,6 @@ = JWT authentication -To enable https://pulsar.apache.org/docs/2.11.x/security-jwt/[[JWT Authentication], set authentication to `true`: +To enable https://pulsar.apache.org/docs/2.11.x/security-jwt/[JWT Authentication], set authentication to `true`: [source,yaml] ---- global: diff --git a/modules/crd-spec/nav.adoc b/modules/crd-spec/nav.adoc deleted file mode 100644 index 8e7aa1d..0000000 --- a/modules/crd-spec/nav.adoc +++ /dev/null @@ -1,9 +0,0 @@ -.CRD spec -* xref:Autorecovery.openapi.adoc[] -* xref:Bastion.openapi.adoc[] -* xref:BookKeeper.openapi.adoc[] -* xref:Broker.openapi.adoc[] -* xref:FunctionsWorker.openapi.adoc[] -* xref:Proxy.openapi.adoc[] -* xref:PulsarCluster.openapi.adoc[] -* xref:ZooKeeper.openapi.adoc[] diff --git a/modules/crd-spec/partials/functionworker.adoc b/modules/crd-spec/partials/functionworker.adoc deleted file mode 100644 index 4272893..0000000 --- a/modules/crd-spec/partials/functionworker.adoc +++ /dev/null @@ -1,17 +0,0 @@ -:crd-namespace: "PulsarCluster.Spec.FunctionsWorker" - -This is an example about functions worker deployment. - -[source, yaml] --- -pulsar-operator: - cluster: - create: true - spec: - global: - name: pulsar - image: datastax/lunastreaming-all:2.10_3.4 - functionsWorker: - replicas: 1 - runtime: kubernetes --- \ No newline at end of file diff --git a/modules/crd-spec/partials/pulsarCluster.adoc b/modules/crd-spec/partials/pulsarCluster.adoc deleted file mode 100644 index ff0265d..0000000 --- a/modules/crd-spec/partials/pulsarCluster.adoc +++ /dev/null @@ -1,14 +0,0 @@ -:crd-namespace: "PulsarCluster" - -This is an example talking about the PulsarCluster object. - -[source, yaml] --- -pulsar-operator: - cluster: - create: true - spec: - global: - name: pulsar - image: datastax/lunastreaming-all:2.10_3.4 --- \ No newline at end of file diff --git a/modules/getting-started/nav.adoc b/modules/getting-started/nav.adoc deleted file mode 100644 index 9716c9a..0000000 --- a/modules/getting-started/nav.adoc +++ /dev/null @@ -1,5 +0,0 @@ -.Get started -* xref:index.adoc[] -* xref:operator.adoc[] -* xref:stack.adoc[] -* xref:upgrades.adoc[] diff --git a/modules/getting-started/pages/index.adoc b/modules/getting-started/pages/index.adoc index fea6620..228d58f 100644 --- a/modules/getting-started/pages/index.adoc +++ b/modules/getting-started/pages/index.adoc @@ -1,4 +1,4 @@ -= Getting started += Get started {pulsar-operator} can be installed in two ways. diff --git a/modules/getting-started/pages/operator.adoc b/modules/getting-started/pages/operator.adoc index 4cab92e..6fdf519 100644 --- a/modules/getting-started/pages/operator.adoc +++ b/modules/getting-started/pages/operator.adoc @@ -1,8 +1,5 @@ = Install {pulsar-operator} Helm chart -// Operator Hub -// The install will be updated when chart is available publicly - {pulsar-operator} is installed using Helm. You can install just the operator and the PulsarCluster CRDs, or you can install xref:stack.adoc[Kaap Stack], which includes the operator, CRDs, and the Prometheus monitoring stack. @@ -16,6 +13,7 @@ helm repo update ---- . The KAAP Operator Helm chart is available for download (https://github.com/datastax/kaap/releases/latest)[here]. + . Install the KAAP operator Helm chart: + [source,shell] @@ -29,45 +27,34 @@ STATUS: deployed REVISION: 1 TEST SUITE: None ---- + . Ensure KAAP operator is up and running: + -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- kubectl get deployment ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- NAME READY UP-TO-DATE AVAILABLE AGE kaap 1/1 1 1 13m ---- --- ==== + . Describe the deployment: + -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- kubectl describe deployment kaap ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- Name: kaap @@ -106,7 +93,7 @@ Pod Template: Environment Variables from: kaap ConfigMap Optional: false Environment: - KUBERNETES_NAMESPACE: (v1:metadata.namespace) + KUBERNETES_NAMESPACE: (v1:metadata.namespace) Mounts: Volumes: Conditions: @@ -121,30 +108,23 @@ Events: ---- ------ ---- ---- ------- Normal ScalingReplicaSet 2m5s deployment-controller Scaled up replica set kaap-76dfd8ddf9 to 1 ---- --- ==== - -. You've now installed KAAP. ++ +You've now installed KAAP. + By default, when KAAP is installed, the PulsarCluster CRDs are also created. This setting is defined in the Pulsar operator values.yaml file as `crd: create: true`. + +. Get the available CRDs: + -. To see the available CRDs: -+ -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- kubectl get crds | grep kaap ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- autorecoveries.kaap.oss.datastax.com 2023-05-12T16:35:59Z @@ -156,34 +136,27 @@ proxies.kaap.oss.datastax.com 2023-05-12T16:36:02Z pulsarclusters.kaap.oss.datastax.com 2023-05-12T16:36:04Z zookeepers.kaap.oss.datastax.com 2023-05-12T16:36:06Z ---- --- ==== == Uninstall -To uninstall the {pulsar-operator}: -[tabs] -==== -Helm:: -+ --- +Uninstall the {pulsar-operator}: + [source,shell] ---- helm uninstall kaap ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- release "kaap" uninstalled ---- --- ==== -== What's next? +== Next steps * xref:authentication:index.adoc[] * xref:scaling-components:autoscale-bookies.adoc[] diff --git a/modules/getting-started/pages/stack.adoc b/modules/getting-started/pages/stack.adoc index ea6dc17..2507773 100644 --- a/modules/getting-started/pages/stack.adoc +++ b/modules/getting-started/pages/stack.adoc @@ -10,24 +10,18 @@ Check out the {pulsar-stack}. * Cert Manager * Keycloak -To install a PulsarCluster with the {pulsar-stack} included: +. Install a PulsarCluster with the {pulsar-stack} included: + -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- helm repo add kaap https://datastax.github.io/kaap helm repo update helm install pulsar kaap/kaap-stack --values helm/examples/dev-cluster/values.yaml ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- NAME: kaap @@ -37,25 +31,18 @@ STATUS: deployed REVISION: 1 TEST SUITE: None ---- --- ==== . Ensure kaap-stack is up and running: + -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- kubectl get deployment ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- NAME READY UP-TO-DATE AVAILABLE AGE @@ -67,25 +54,18 @@ pulsar-kube-prometheus-sta-operator 1/1 1 1 5m19s pulsar-kube-state-metrics 1/1 1 1 5m19s pulsar-proxy 1/1 1 1 3m19s ---- --- ==== . Describe the deployment: + -[tabs] -==== -Kubectl:: -+ --- [source,shell] ---- kubectl describe deployment kaap-stack ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- Name: kaap @@ -139,35 +119,29 @@ Events: ---- ------ ---- ---- ------- Normal ScalingReplicaSet 12m deployment-controller Scaled up replica set kaap-76dfd8ddf9 to 1 ---- --- ==== You've now installed {pulsar-stack}. == Uninstall -To uninstall the KAAP operator and the cluster: -[tabs] -==== -Helm:: -+ --- + +Uninstall the KAAP operator and the cluster: + [source,shell] ---- helm uninstall kaap-stack ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- release "kaap-stack" uninstalled ---- --- ==== -== What's next? +== Next steps * xref:authentication:index.adoc[] * xref:scaling-components:autoscale-bookies.adoc[] diff --git a/modules/getting-started/pages/upgrades.adoc b/modules/getting-started/pages/upgrades.adoc index 493575f..dab12df 100644 --- a/modules/getting-started/pages/upgrades.adoc +++ b/modules/getting-started/pages/upgrades.adoc @@ -1,10 +1,14 @@ = Upgrade deployments The {pulsar-operator} performs cluster upgrades in a very conservative manner, with the primary goal of reducing maintenance time during upgrades. -Components are updated and then restarted *only* if strictly needed. For example, if only the broker needs to be upgraded, then all other services will be left up and running. +Components are updated and then restarted *only* if strictly needed. +For example, if only the broker needs to be upgraded, then all other services will be left up and running. If there is an error or interruption during upgrade, the operator will apply the desired state defined in the PulsarCluster custom resource until the resource matches the actual state. +== Upgrade schema + The {pulsar-operator} follows a fixed schema to upgrade the cluster: + [source,console] ---- stateDiagram-v2 @@ -32,23 +36,18 @@ stateDiagram-v2 ar --> [*] : Ready ---- -== Example +== Upgrade example + +For this example, assume you installed the operator and a Pulsar cluster with the following yaml file: -For example, if you've installed the operator and a Pulsar cluster with this yaml file: -[tabs] -==== -Helm:: -+ --- [source,shell] ---- helm install pulsar helm/pulsar-operator -f helm/examples/bookie-autoscaling/values.yaml ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- NAME: pulsar @@ -58,12 +57,10 @@ STATUS: deployed REVISION: 1 TEST SUITE: None ---- --- ==== -. While looking over your configuration, you decide you also need more brokers to handle requests. -Edit the helm/examples/bookie-autoscaling/values.yaml file to change the number of Brokers from 1 to 3: -+ +If you want to add more brokers to handle requests, edit `helm/examples/bookie-autoscaling/values.yaml` to change the `broker.replicas` and `broker.config.managedLedgerDefault*` properties from `1` to `3`: + [source,shell] ---- broker: @@ -78,22 +75,16 @@ broker: memory: "128Mi" ---- -. Upgrade the cluster with the new values.yaml file: -+ -[tabs] -==== -Helm:: -+ --- +Then, upgrade the cluster with the new `values.yaml` file: + [source,shell] ---- helm upgrade pulsar helm/pulsar-operator -f helm/examples/bookie-autoscaling/values.yaml ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- Release "pulsar" has been upgraded. Happy Helming! @@ -104,25 +95,19 @@ STATUS: deployed REVISION: 5 TEST SUITE: None ---- --- ==== -. The operator will upgrade the cluster to the new desired state. In this case, the operator adds two more broker pods, and touches nothing else. -+ -[tabs] -==== -Kubectl:: -+ --- +The operator upgrades the cluster to the new desired state. +In this case, the operator adds two more broker pods, and touches nothing else. + [source,shell] ---- kubectl get pods -A ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- pulsar-cluster pulsar-autorecovery-57cfc8b84d-xvchb 1/1 Running 0 150m @@ -136,28 +121,22 @@ pulsar-cluster pulsar-proxy-5bd5cc4fb6-9pnl7 1/1 pulsar-cluster pulsar-zookeeper-0 1/1 Running 0 155m pulsar-cluster pulsar-zookeeper-metadata-zgfn4 0/1 Completed 0 153m ---- --- ==== -. You've successfully upgraded your deployment by just managing a single .yaml file. Thanks, {pulsar-operator}! +You've successfully upgraded your deployment by changing only one YAML file. == Upgrade CRDs -To upgrade CRDs: -[tabs] -==== -Helm:: -+ --- +To upgrade CRDs, run the following command: + [source,shell] ---- kubectl replace -f helm/pulsar-operator/crds ---- --- -Result:: -+ --- +.Result +[%collapsible] +==== [source,console] ---- customresourcedefinition.apiextensions.k8s.io/autorecoveries.pulsar.oss.datastax.com replaced @@ -169,18 +148,10 @@ customresourcedefinition.apiextensions.k8s.io/proxies.pulsar.oss.datastax.com re customresourcedefinition.apiextensions.k8s.io/pulsarclusters.pulsar.oss.datastax.com replaced customresourcedefinition.apiextensions.k8s.io/zookeepers.pulsar.oss.datastax.com replaced ---- --- ==== -== What's next? +== Next steps * xref:authentication:index.adoc[] * xref:scaling-components:autoscale-bookies.adoc[] -* xref:resource-sets:index.adoc[] - - - - - - - +* xref:resource-sets:index.adoc[] \ No newline at end of file diff --git a/modules/migration/nav.adoc b/modules/migration/nav.adoc deleted file mode 100644 index bae4deb..0000000 --- a/modules/migration/nav.adoc +++ /dev/null @@ -1,2 +0,0 @@ -.Migrate -* xref:migrate-cluster.adoc[] diff --git a/modules/migration/pages/migrate-cluster.adoc b/modules/migration/pages/migrate-cluster.adoc index 755a33b..ee94a18 100644 --- a/modules/migration/pages/migrate-cluster.adoc +++ b/modules/migration/pages/migrate-cluster.adoc @@ -12,6 +12,7 @@ You can then examine the report and decide if you want to proceed with the clust * Migration-tool JAR downloaded from the https://github.com/datastax/kaap/releases[latest release]. == Scan and generate cluster CRDs + . Create an input file called `input-cluster-specs.yaml` with the following content: + [source,yaml] @@ -21,57 +22,54 @@ namespace: clusterName: ---- -.. To retrieve the context-name: -+ -[tabs] -==== -Helm:: +.. To retrieve the `context`: + --- [source,shell] ---- kubectl config get-contexts ---- --- - -Result:: + --- +.Result +[%collapsible] +==== [source,console] ---- CURRENT NAME CLUSTER AUTHINFO NAMESPACE * gke_gcp-techpubs_us-east1-b_pulsar-cluster gke_gcp-techpubs_us-east1-b_pulsar-cluster gke_gcp-techpubs_us-east1-b_pulsar-cluster pulsar-cluster ---- --- ==== + -[NOTE] -==== -We recommend switching to K8's current context and ensuring connectivity (`kubectl get pods`, for example) before running the migration tool. -==== +Before running the migration tool, {company} recommends switching to K8's current context and ensuring connectivity, such as with `kubectl get pods`. + +.. The `namespace` is the namespace with the Apache Pulsar resources you wish to scan. -.. The namespace is the namespace with the Apache Pulsar resources you wish to scan. -.. The clusterName is the prefix of each pod. For example, if the broker pod is `pulsar-prod-cluster-broker-0`, the `clusterName` is `pulsar-prod-cluster`. +.. The `clusterName` is the prefix of each pod. +For example, if the broker pod is `pulsar-prod-cluster-broker-0`, then the `clusterName` is `pulsar-prod-cluster`. -. After you've added the retrieved values to input-cluster-specs.yaml, generate the report with: +. After you've added the retrieved values to `input-cluster-specs.yaml`, generate the report: + [source,java] ---- java -jar migration-tool.jar generate -i input-cluster-specs.yaml -o output ---- -. You'll find a link to the generated report in the logs. -Open the generated report in your browser and examine the differences between the existing cluster and the KAAP operator. -If everything looks good, proceed to the migration procedure. -If you find you need to change the generated CRD and simulate the migration again, run: +. Find the link to the generated report in the logs, open the generated report in your browser, and then examine the differences between the existing cluster and the KAAP operator. ++ +If everything looks good, proceed to the <>. ++ +If you need to change the generated CRD and simulate the migration again, run the following command after making the necessary changes: + [source,java] ---- java -jar migration-tool.jar diff -d output/ ---- +[#migration-procedure] == Migration procedure -. Create a new `values.yaml` file for the operator. Then in the `pulsar-operator.cluster` section, copy the generated CRD's spec. + +. Create a new `values.yaml` file for the operator. + +. In the `pulsar-operator.cluster` section, enter the generated CRD's spec. + [source,yaml] ---- @@ -80,47 +78,45 @@ pulsar-operator: create: true name: spec: - + ---- -. Install the operator release with the above values file. +. Install the operator release with the new `values.yaml` file: + [source,shell] ---- helm install pulsar kaap/kaap-stack --values ---- -. Wait for the PulsarCluster status to be in a Ready state, indicating the operator has assumed control of the cluster. -Since the generated resources will match the existing cluster's resources, the following behaviours are expected: - -* The operator will not create any new resources. -* The operator will not delete any existing resources. -* The operator will restart every statefulset/deployment. This will be done in a safe manner using the staged upgrades feature of the operator. +. Wait for the PulsarCluster status to be in a Ready state, which indicates that the operator has assumed control of the cluster. +Since the generated resources will match the existing cluster's resources, the following behaviors are expected: ++ +* The operator doesn't create any new resources. +* The operator doesn't delete any existing resources. +* The operator doesn't restart every statefulset/deployment. +This is done in a safe manner using the operator's staged upgrades feature. == Cleanup -. Delete the existing chart release. +. Delete the existing chart release: + [source,shell] ---- kubectl delete secret -l name=,owner=helm ---- -. Clean up Helm annotations and labels. - -.. You can safely remove the following annotations: - -* meta.helm.sh/release-name -* meta.helm.sh/release-namespace - -.. You can safely remove the following labels: - -* app.kubernetes.io/managed-by -* chart -* release -* heritage - -== What's next? +. Clean up Helm annotations and labels: ++ +* You can safely remove the following annotations: +** `meta.helm.sh/release-name` +** `meta.helm.sh/release-namespace` +* You can safely remove the following labels: +** `app.kubernetes.io/managed-by` +** `chart` +** `release` +** `heritage` + +== Next steps Now that you've migrated your existing cluster to the operator, you can start using the operator's features to manage your cluster. diff --git a/modules/resource-sets/nav.adoc b/modules/resource-sets/nav.adoc deleted file mode 100644 index 9be0b5a..0000000 --- a/modules/resource-sets/nav.adoc +++ /dev/null @@ -1,6 +0,0 @@ -.Resource sets -* xref:index.adoc[] -* xref:bookies.adoc[] -* xref:pods.adoc[] -* xref:proxies.adoc[] -* xref:racks.adoc[] diff --git a/modules/resource-sets/pages/index.adoc b/modules/resource-sets/pages/index.adoc index 88d672a..fe9911e 100644 --- a/modules/resource-sets/pages/index.adoc +++ b/modules/resource-sets/pages/index.adoc @@ -17,5 +17,5 @@ Racks, proxies, bookies, and pods can likewise be created as resource sets with [source,shell] ---- helm install pulsar-operator helm/pulsar-operator \ - --values helm/examples/resource-sets/values.yaml + --values helm/examples/resource-sets/values.yaml ---- \ No newline at end of file diff --git a/modules/scaling-components/nav.adoc b/modules/scaling-components/nav.adoc deleted file mode 100644 index 0ca6303..0000000 --- a/modules/scaling-components/nav.adoc +++ /dev/null @@ -1,5 +0,0 @@ -.Scaling components -* xref:index.adoc[] -* xref:autoscale-bookies.adoc[] -* xref:autoscale-brokers.adoc[] -* xref:kafka.adoc[] diff --git a/modules/scaling-components/pages/autoscale-bookies.adoc b/modules/scaling-components/pages/autoscale-bookies.adoc index 120d87a..34886fa 100644 --- a/modules/scaling-components/pages/autoscale-bookies.adoc +++ b/modules/scaling-components/pages/autoscale-bookies.adoc @@ -8,16 +8,21 @@ When KAAP sees low storage usage on a Bookkeeper node, the node is automatically [source,shell] ---- helm install pulsar-operator helm/pulsar-operator \ - --values helm/examples/bookie-autoscaling/values.yaml + --values helm/examples/bookie-autoscaling/values.yaml ---- == Bookkeeper autoscaler configuration The operator will scale the number of bookies pods in a cluster up and down based on current disk usage. -The operator checks the disk usage percentage of all bookies at a regular interval. If all of the bookies' memory is over a percentage threshold, the operator will add bookies, and if under the low threshold, will decommission a bookie to save resources. +The operator checks the disk usage percentage of all bookies at a regular interval. + +If all of the bookies' memory is over a percentage threshold, the operator will add bookies, and if under the low threshold, will decommission a bookie to save resources. + When a bookie's memory is underutilized, you want to decommission it to save resources. -The scaling behavior is similar to the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Kubernetes Horizontal Pod Autoscaler. -The operator's thresholds are set in the values.yaml file. + +The scaling behavior is similar to the https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/[Kubernetes Horizontal Pod Autoscaler]. + +The operator's thresholds are set in the values.yaml file: + [source,shell] ---- bookkeeper: @@ -32,6 +37,7 @@ The operator's thresholds are set in the values.yaml file. + scaleDownBy: 1 stabilizationWindowMs: 30000 ---- + .Bookkeeper autoscaler configuration [cols=4*,options="header"] |=== @@ -89,9 +95,11 @@ The operator's thresholds are set in the values.yaml file. + == Test bookie autoscaler Once you've deployed a Pulsar cluster with bookie autoscaling enabled, test it by adding load to the cluster and watching the operator pod's logs. -[NOTE] + +[TIP] ==== -If you don't have a bastion pod, you can add it to your cluster in the same values.yaml file you used to deploy the operator. +If you don't have a bastion pod, you can add it to your cluster in the same `values.yaml` file you used to deploy the operator: + [source,shell] ---- bastion: @@ -103,40 +111,31 @@ bastion: ---- ==== -. Exec into your bastion pod. +. Exec into your bastion pod: + [source,shell] ---- kubectl exec --stdin --tty -- /bin/bash ---- -. Run a https://pulsar.apache.org/docs/performance-pulsar-perf/[Pulsar perf] test in your deployment, and follow the operator's logs to see the autoscaler in action. +. Run a https://pulsar.apache.org/docs/performance-pulsar-perf/[Pulsar perf] test in your deployment, and then follow the operator's logs to see the autoscaler in action: + -[tabs] -==== -Bastion pod:: -+ --- [source,shell] ---- bin/pulsar-perf produce topic ---- --- - -Result:: + --- +.Result [source,console] ---- 2023-05-19T14:39:34,726+0000 [pulsar-perf-producer-exec-1-1] INFO org.apache.pulsar.testclient.PerformanceProducer - Created 1 producers 2023-05-19T14:39:34,778+0000 [pulsar-client-io-2-1] INFO com.scurrilous.circe.checksum.Crc32cIntChecksum - SSE4.2 CRC32C provider initialized 2023-05-19T14:39:43,190+0000 [main] INFO org.apache.pulsar.testclient.PerformanceProducer - Throughput produced: 817 msg --- 81.7 msg/s --- 0.6 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 12.008 ms - med: 10.571 - 95pct: 20.821 - 99pct: 32.194 - 99.9pct: 46.759 - 99.99pct: 56.243 - Max: 56.243 ---- --- -==== - -. The operator notices the differing values and patches the bookkeeper-set to keep up with the increased memory usage of the bookkeeper pods. + +The operator notices the differing values and patches the bookkeeper-set to keep up with the increased memory usage of the bookkeeper pods. ++ +.Result [source,console] ---- | 14:41:07 INFO [com.dat.oss.pul.crd.SpecDiffer] (ReconcilerExecutor-pulsar-bk-controller-69) 'bookkeeper.replicas' value differs: @@ -147,8 +146,12 @@ Result:: │ 14:41:07 INFO [com.dat.oss.pul.con.boo.BookKeeperResourcesFactory] (ReconcilerExecutor-pulsar-bk-controller-69) Cleaning up orphan PVCs for bookie-s ---- -. Cancel the Pulsar perf test with Ctrl-C. The operator will notice the decreased load and scale down the number of bookies. Notice that the operator scales down the number of bookies by 1 at a time, as specified in the `scaleDownBy` parameter, and properly decommissions them. +. Cancel the Pulsar perf test with Ctrl-C. ++ +The operator will notice the decreased load and scale down the number of bookies. +Notice that the operator scales down the bookies one by one, as specified in the `scaleDownBy` parameter, and properly decommissions them: + +.Result [source,console] ---- │ 15:32:19 INFO [com.dat.oss.pul.aut.BookKeeperSetAutoscaler] (pool-9-thread-1) isDiskUsageAboveTolerance: false for pulsar-bookkeeper-8 (BookieAdminClient.BookieLed │ @@ -158,4 +161,4 @@ Result:: │ 15:32:19 INFO [com.dat.oss.pul.aut.boo.PodExecBookieAdminClient] (OkHttp https://10.12.0.1/...) Bookie pulsar-bookkeeper-9 is set to read-only=true │ │ 15:32:22 INFO [com.dat.oss.pul.aut.boo.BookieDecommissionUtil] (ReconcilerExecutor-pulsar-bk-controller-74) Attempting decommission of bookie pulsar-bookkeeper-9 w │ │ 15:32:22 INFO [com.dat.oss.pul.aut.boo.PodExecBookieAdminClient] (ReconcilerExecutor-pulsar-bk-controller-74) Starting bookie recovery for bookie pulsar-bookkeeper │ ----- +---- \ No newline at end of file diff --git a/modules/scaling-components/pages/autoscale-brokers.adoc b/modules/scaling-components/pages/autoscale-brokers.adoc index 24bccb6..14b460e 100644 --- a/modules/scaling-components/pages/autoscale-brokers.adoc +++ b/modules/scaling-components/pages/autoscale-brokers.adoc @@ -1,18 +1,27 @@ = Broker autoscaler The operator scales the number of broker pods in a cluster up and down based on current CPU usage. -The CPU usage of each broker is checked at the Pulsar load balancer, not just at the Kubenetes pod level. This means that the operator can scale brokers based on the CPU usage of all brokers in the cluster, not just the CPU usage of a single broker pod. +The CPU usage of each broker is checked at the Pulsar load balancer, not just at the Kubenetes pod level. +This means that the operator can scale brokers based on the CPU usage of all brokers in the cluster, not just the CPU usage of a single broker pod. + When the operator sees that the Pulsar load balancer is having trouble finding brokers to assign topic bundles to, it will scale up the number of brokers to handle the load. When the operator sees that the CPU usage of all brokers is low, it will scale down the number of brokers to save resources. -CPU usage is tightly coupled to traffic, so you can expect to see significant scaling activity with broker autoscaler enabled. This value can be controlled with the `stabilizationWindowMs` parameter, which tells the operator how long to wait between scaling events. + +CPU usage is tightly coupled to traffic, so you can expect to see significant scaling activity with broker autoscaler enabled. +This value can be controlled with the `stabilizationWindowMs` parameter, which tells the operator how long to wait between scaling events. == Install Operator with broker autoscaler enabled + +Use this command to install the operator with broker autoscaling enabled: + [source,shell] ---- helm install pulsar-operator helm/pulsar-stack \ --values helm/examples/broker-autoscaling/values.yaml ---- -The operator's thresholds are set in the values.yaml file. + + +The operator's thresholds are set in the `values.yaml` file: + [source,shell] ---- broker: @@ -28,6 +37,7 @@ The operator's thresholds are set in the values.yaml file. + scaleDownBy: 1 stabilizationWindowMs: 60000 ---- + .Broker autoscaler configuration [cols=4*,options="header"] |=== @@ -85,9 +95,11 @@ The operator's thresholds are set in the values.yaml file. + == Test broker autoscaler Once you've deployed a Pulsar cluster with broker autoscaling enabled, test it by adding load to the cluster and watching the operator pod's logs. -[NOTE] + +[TIP] ==== -If you don't have a bastion pod, you can add it to your cluster in the same values.yaml file you used to deploy the operator. +If you don't have a bastion pod, you can add it to your cluster in the same `values.yaml` file you used to deploy the operator: + [source,shell] ---- bastion: @@ -99,62 +111,56 @@ bastion: ---- ==== -. Exec into your bastion pod. +. Exec into your bastion pod: + [source,shell] ---- kubectl exec --stdin --tty -- /bin/bash ---- -. Run a https://pulsar.apache.org/docs/performance-pulsar-perf/[Pulsar perf] test in your deployment, and follow the operator's logs to see the autoscaler in action. -+ -[tabs] -==== -Bastion pod:: +. Run a https://pulsar.apache.org/docs/performance-pulsar-perf/[Pulsar perf] test in your deployment, and then follow the operator's logs to see the autoscaler in action: + --- [source,shell] ---- bin/pulsar-perf produce topic ---- --- - -Result:: + --- +.Result [source,console] ---- 2023-05-19T14:39:34,726+0000 [pulsar-perf-producer-exec-1-1] INFO org.apache.pulsar.testclient.PerformanceProducer - Created 1 producers 2023-05-19T14:39:34,778+0000 [pulsar-client-io-2-1] INFO com.scurrilous.circe.checksum.Crc32cIntChecksum - SSE4.2 CRC32C provider initialized 2023-05-19T14:39:43,190+0000 [main] INFO org.apache.pulsar.testclient.PerformanceProducer - Throughput produced: 817 msg --- 81.7 msg/s --- 0.6 Mbit/s --- failure 0.0 msg/s --- Latency: mean: 12.008 ms - med: 10.571 - 95pct: 20.821 - 99pct: 32.194 - 99.9pct: 46.759 - 99.99pct: 56.243 - Max: 56.243 ---- --- -==== - -. The operator notices the differing values and patches the broker-set to keep up with the increased CPU usage of the broker pods. + +The operator notices the differing values and patches the broker-set to keep up with the increased CPU usage of the broker pods: ++ +.Result [source,console] ---- 17:06:09 INFO [com.dat.oss.pul.aut.bro.LoadReportResourceUsageSource] (pool-9-thread-1) Broker pulsar-broker-0 cpu usage: 7.000000 % │ 17:06:10 INFO [com.dat.oss.pul.aut.bro.LoadReportResourceUsageSource] (pool-9-thread-1) Broker pulsar-broker-1 cpu usage: 15.000001 % │ 17:06:10 INFO [com.dat.oss.pul.aut.BrokerSetAutoscaler] (pool-9-thread-1) Scaled brokers for broker set broker from 2 to 3 │ 17:06:10 INFO [com.dat.oss.pul.crd.SpecDiffer] (ReconcilerExecutor-pulsar-broker-controller-67) 'broker.replicas' value differs: -was:2 +was: 2 now: 3 ---- + -[NOTE] -==== -The default for lowerCpuThreshold is `0.4` and higherCpuThreshold is `0.8`. You may need to set these values lower in values.yaml to trigger broker scaling. -==== -. Cancel the Pulsar perf test with Ctrl-C. The operator will notice the decreased load and scale down the number of brokers. Notice that the operator scales down the number of brokers by 1 at a time, as specified in the `scaleDownBy` parameter, and properly decommissions them. +The default for `lowerCpuThreshold` is `0.4`, and the default for `higherCpuThreshold` is `0.8`. +You might need to reduce these values in `values.yaml` to trigger broker scaling. + +. Cancel the Pulsar perf test with Ctrl-C. + +The operator will notice the decreased load and scale down the number of brokers. +Notice that the operator scales down the brokers one by one, as specified in the `scaleDownBy` parameter, and properly decommissions them: ++ +.Result [source,console] ---- 17:18:39 INFO [com.dat.oss.pul.aut.bro.LoadReportResourceUsageSource] (pool-9-thread-1) Broker pulsar-broker-6 cpu usage: 2.000000 % 17:18:39 INFO [com.dat.oss.pul.aut.BrokerSetAutoscaler] (pool-9-thread-1) Scaled brokers for broker set broker from 7 to 6 17:18:39 INFO [com.dat.oss.pul.crd.SpecDiffer] (ReconcilerExecutor-pulsar-broker-controller-74) 'broker.replicas' value differs: was: 7 -now: 6 │ +now: 6 │ 17:18:40 INFO [com.dat.oss.pul.con.AbstractResourceSetsController] (ReconcilerExecutor-pulsar-broker-controller-74) broker-set 'broker' patched ----- +---- \ No newline at end of file diff --git a/modules/scaling-components/pages/kafka.adoc b/modules/scaling-components/pages/kafka.adoc index 26f796a..30a78ae 100644 --- a/modules/scaling-components/pages/kafka.adoc +++ b/modules/scaling-components/pages/kafka.adoc @@ -3,7 +3,7 @@ Have an Apache Kafka(R) workload you want to control with {pulsar-operator}? Thanks to Starlight for Kafka, you can run your Kafka workload on a Pulsar cluster, and with {pulsar-operator}, the scaling of the Kafka pods is handled for you. -== Scaling the Pulsar Broker with a Kafka Client Workload +== Scale the Pulsar Broker with a Kafka Client Workload This folder contains a sample configuration and demo about how to run a workload on an Apache Pulsar(R) cluster with the Broker Auto Scaling feature. @@ -84,7 +84,7 @@ kubectl apply -f helm/examples/kafka/kafka-consumer-perf.yaml kubectl logs -f deploy/kafka-client-consumer ---- -== Scaling the client workloads +== Scale the client workloads . Open a terminal and view the logs of the Operator. + @@ -97,7 +97,7 @@ kubectl logs -f deploy/pulsar-operator + [source,shell] ---- -kubectl scale deploy/kafka-client-consumer --replicas 10 +kubectl scale deploy/kafka-client-consumer --replicas 10 ---- As the load increases you will see the Operator scales out the Broker STS. \ No newline at end of file diff --git a/operator-hub-description.adoc b/operator-hub-description.adoc deleted file mode 100644 index 3fef640..0000000 --- a/operator-hub-description.adoc +++ /dev/null @@ -1,21 +0,0 @@ -= K8s Autoscaler for Apache Pulsar (KAAP) - -Kubernetes Autoscaling for Apache Pulsar (KAAP) simplifies running [Apache Pulsar](https://pulsar.apache.org) on Kubernetes by applying the familiar [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to Pulsar's components, and horizonally scaling resources up or down based on CPU and memory workloads. - -Full documentation is available in the https://docs.datastax.com/en/streaming/kaap-operator/latest/index.html[DataStax Streaming Documentation]. - -== Apache Pulsar - -Apache Pulsar is an open-source distributed pub-sub messaging platform designed to handle real-time data streams and event-driven architectures. It offers a highly scalable and durable messaging system that allows seamless communication and processing of data among diverse applications and services. Pulsar provides advanced features like multi-tenancy, geo-replication, message replay, and schema enforcement, enabling organizations to build robust, reliable, and flexible data streaming pipelines. With its horizontally scalable architecture and extensive ecosystem integrations, Apache Pulsar empowers businesses to efficiently manage and process massive amounts of data while ensuring low-latency and high-throughput messaging capabilities. - -== KAAP Operator - -Operating and maintaining Apache Pulsar clusters traditionally involves complex manual configurations, making it challenging for developers and operators to effectively manage the system's lifecycle. However, with the Apache Pulsar Kubernetes Operator, these complexities are abstracted away, enabling developers to focus on their applications rather than the underlying infrastructure. - -This operator acts as an extension to the Kubernetes API, introducing custom resource definitions (CRDs) specific to Apache Pulsar. With these CRDs, users can define Pulsar clusters as native Kubernetes objects. The operator then reconciles the desired state defined by these objects with the actual state of the Pulsar cluster, ensuring that the cluster is always in the desired configuration. - -After a new custom resource type is added to your cluster by installing a CRD, you can create instances of the resource based on its specification. The Kubernetes API can be extended to support the new resource type, automating away the tedious aspects of managing a Pulsar cluster. - -== Details - -KAAP is configured, deployed, and packaged with Helm charts and based on the https://quarkiverse.github.io/quarkiverse-docs/quarkus-operator-sdk/dev/index.html[Quarkus Operator SDK]. \ No newline at end of file