chore/kraft migration

CHANGELOG.md

@@ -12,6 +12,9 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Refactor: move server configuration properties from the command line to configuration files. ([#911]).
+- BREAKING: add support for ZooKeeper to KRaft migration ([#923]).
+  In order to support migration to Kraft, the operator must disable automatic Kafka broker id generation.
+  The broker ids generated by the operator are incompatible with those generated by Kafka.
 - Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#925]).
 
 ### Removed

deploy/helm/kafka-operator/crds/crds.yaml

@@ -787,6 +787,62 @@ spec:
                             - configMapName
                           type: object
                       type: object
+                    brokerIdPodConfigMapName:
+                      description: |-
+                        Enable users to manually assign Kafka broker ids.
+
+                        Name of a ConfigMap containing a mapping of broker IDs to pod names.
+                        The ConfigMap must contain a key for every broker pod in the cluster with the following format:
+                        `<pod-name>: <broker-id>`
+
+                        Example:
+                        ```
+                        ---
+                        apiVersion: v1
+                        kind: ConfigMap
+                        metadata:
+                          name: brokeridmapping
+                        data:
+                          simple-kafka-broker-default-0: "2001"
+                          simple-kafka-broker-default-1: "2001"
+                          simple-kafka-broker-default-2: "2002"
+                        ```
+                        This is necessary when migrating from ZooKeeper to Kraft mode to retain existing broker IDs
+                        because previously broker ids were generated by Kafka and not the operator.
+                      nullable: true
+                      type: string
+                    metadataManager:
+                      description: |-
+                        Metadata manager to use for the Kafka cluster.
+
+                        IMPORTANT: This property will be removed as soon as Kafka 3.x support is dropped.
+
+                        Possible values are `zookeeper` and `kraft`.
+
+                        If not set, defaults to:
+
+                        - `zookeeper` for Kafka versions below `4.0.0`.
+                        - `kraft` for Kafka versions `4.0.0` and higher.
+
+                        Using `zookeeper` for Kafka versions `4.0.0` and higher is not supported.
+
+                        When set to `kraft`, the operator will perform the following actions:
+
+                        * Generate the Kafka cluster id.
+                        * Assign broker roles and configure controller quorum voters in the `broker.properties` files.
+                        * Format storage before (re)starting Kafka brokers.
+                        * Remove ZooKeeper related configuration options from the `broker.properties` files.
+
+                        These actions are **mandatory** when in Kraft mode and partially exclusive to the ZooKeeper mode.
+                        This means they **cannot** be performed in ZooKeeper mode.
+
+                        This property is also useful when migrating from ZooKeeper to Kraft mode because it permits the operator
+                        to reconcile controllers while still using ZooKeeper for brokers.
+                      enum:
+                        - zookeeper
+                        - kraft
+                      nullable: true
+                      type: string
                     tls:
                       default:
                         internalSecretClass: tls
@@ -830,7 +886,7 @@ spec:
                         Provide the name of the ZooKeeper [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery)
                         here. When using the [Stackable operator for Apache ZooKeeper](https://docs.stackable.tech/home/nightly/zookeeper/)
                         to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource.
-                        This can only be used up to Kafka version 3.9.x. Since Kafka 4.0.0, ZooKeeper suppport was dropped.
+                        This can only be used up to Kafka version 3.9.x. Since Kafka 4.0.0, ZooKeeper support was dropped.
                         Please use the 'controller' role instead.
                       nullable: true
                       type: string

docs/modules/kafka/examples/kraft_migration/01-setup.yaml

@@ -0,0 +1,99 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    stackable.tech/vendor: Stackable
+  name: kraft-migration
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: simple-zk
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.4
+    pullPolicy: IfNotPresent
+  servers:
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name: simple-kafka-znode
+  namespace: kraft-migration
+spec:
+  clusterRef:
+    name: simple-zk
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: kafka-internal-tls
+spec:
+  backend:
+    autoTls:
+      ca:
+        secret:
+          name: secret-provisioner-kafka-internal-tls-ca
+          namespace: kraft-migration
+        autoGenerate: true
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+  name: kafka-client-auth-tls
+spec:
+  provider:
+    tls:
+      clientCertSecretClass: kafka-client-auth-secret
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: kafka-client-auth-secret
+spec:
+  backend:
+    autoTls:
+      ca:
+        secret:
+          name: secret-provisioner-tls-kafka-client-ca
+          namespace: kraft-migration
+        autoGenerate: true
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: broker-ids
+  namespace: kraft-migration
+data:
+  map.csv: |
+    2000,simple-kafka-broker-default-0
+    2001,simple-kafka-broker-default-1
+    2002,simple-kafka-broker-default-2
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: zookeeper
+    brokerIdPodConfigMapName: broker-ids
+    authentication:
+      - authenticationClass: kafka-client-auth-tls
+    tls:
+      internalSecretClass: kafka-internal-tls
+      serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+  brokers:
+    roleGroups:
+      default:
+        replicas: 3

docs/modules/kafka/examples/kraft_migration/02-start-controllers.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: zookeeper
+    authentication:
+      - authenticationClass: kafka-client-auth-tls
+    tls:
+      internalSecretClass: kafka-internal-tls
+      serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+    brokerIdPodConfigMapName: broker-ids
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    roleGroups:
+      default:
+        replicas: 3
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    configOverrides:
+      controller.properties:
+        zookeeper.metadata.migration.enable: "true" # Enable migration mode so the controller can read metadata from ZooKeeper.

docs/modules/kafka/examples/kraft_migration/03-migrate-metadata.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: zookeeper
+    authentication:
+      - authenticationClass: kafka-client-auth-tls
+    tls:
+      internalSecretClass: kafka-internal-tls
+      serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+    brokerIdPodConfigMapName: broker-ids
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    roleGroups:
+      default:
+        replicas: 3
+    configOverrides:
+      broker.properties:
+        inter.broker.protocol.version: "3.9" # - Latest value known to Kafka 3.9.1
+        zookeeper.metadata.migration.enable: "true" # - Enable migration mode so the broker can participate in metadata migration.
+        controller.listener.names: "CONTROLLER"
+        controller.quorum.voters: "2110489703@simple-kafka-controller-default-0.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093,2110489704@simple-kafka-controller-default-1.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093,2110489705@simple-kafka-controller-default-2.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093"
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    configOverrides:
+      controller.properties:
+        zookeeper.metadata.migration.enable: "true" # Enable migration mode so the controller can read metadata from ZooKeeper.

docs/modules/kafka/examples/kraft_migration/04-migrate-brokers.yaml

@@ -0,0 +1,40 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: zookeeper
+    authentication:
+      - authenticationClass: kafka-client-auth-tls
+    tls:
+      internalSecretClass: kafka-internal-tls
+      serverSecretClass: tls
+    zookeeperConfigMapName: simple-kafka-znode
+    brokerIdPodConfigMapName: broker-ids
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    roleGroups:
+      default:
+        replicas: 3
+    configOverrides:
+      broker.properties:
+        controller.listener.names: "CONTROLLER"
+        controller.quorum.voters: "2110489703@simple-kafka-controller-default-0.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093,2110489704@simple-kafka-controller-default-1.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093,2110489705@simple-kafka-controller-default-2.simple-kafka-controller-default-headless.kraft-migration.svc.cluster.local:9093"
+        process.roles: "broker"
+        node.id: "${env:REPLICA_ID}"
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    configOverrides:
+      controller.properties:
+        zookeeper.metadata.migration.enable: "true" # Enable migration mode so the controller can read metadata from ZooKeeper.

docs/modules/kafka/examples/kraft_migration/05-kraft-mode.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+  namespace: kraft-migration
+spec:
+  image:
+    productVersion: 3.9.1
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    metadataManager: kraft
+    authentication:
+      - authenticationClass: kafka-client-auth-tls
+    tls:
+      internalSecretClass: kafka-internal-tls
+      serverSecretClass: tls
+    brokerIdPodConfigMapName: broker-ids
+  brokers:
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"
+    roleGroups:
+      default:
+        replicas: 3
+    configOverrides:
+      broker.properties:
+        controller.listener.names: "CONTROLLER"
+  controllers:
+    roleGroups:
+      default:
+        replicas: 3
+    envOverrides:
+      KAFKA_CLUSTER_ID: "aC1zl524Svm_uIjcvUGWSw"