kafka/variables.yml

---

# Name of this instance
# Only usefull if you intent to run several instances on your cluster (in the same namespace)
instance: kafka

vault:
  # PKI settings
  pki:
    path: "[[ .vault.root ]]pki/kafka"
    ou: Kafka Cluster ([[ .instance ]])

kafka:
  broker:
    # Number of brokers to run
    count: 3

    # Version of Kafka to use to build the image
    version: 3.7.0

    # Docker image to use
    image: '[[ .docker.repo ]]kafka-broker:[[ .kafka.broker.version ]]-4'

    # Custom env var to set in the container
    env: {}

    # Resource allocation for each broker
    resources:
      cpu: 100
      memory: 1024

    # Prometheus settings
    prometheus:
      enabled: '[[ .prometheus.available ]]'
      metrics_url: http://127.0.0.1:9404/metrics

    # You can set static ports for both client and broker port. If not set, ports will be allocated dynamicaly by Nomad
    # static_client_port: 9092
    # static_broker_port: 9095
    # Ensure all broker runs on distinct hosts
    nomad:
      constraints:
        - operator: distinct_hosts
          value: true

    # Consul settings
    consul:
      meta:
        # Push the name of the broker as a meta of the service
        broker: 'broker-${NOMAD_ALLOC_INDEX}.[[ .instance ]]-broker[[ .consul.suffix ]].service.[[ .consul.domain ]]'
      # Service mesh
      connect:
        upstreams:
          # Default is to contact a zookeeper cluster through the mesh
          - destination_name: zookeeper[[ .consul.suffix ]]
            local_bind_port: 2181

    # Vault settings
    vault:
      # Policies to attach to the task
      policies:
        - '[[ .instance ]]-broker[[ .consul.suffix ]]'

    # Wait for ZooKeeper to be available before starting brokers
    wait_for:
      - service: zookeeper[[ .consul.suffix ]]
        count: 2

    # ZpooKeeper settings
    zookeeper:
      servers:
        - 127.0.0.1:2181
      # path: /kafka
      user: ""
      password: ""

    # If set to false, ACL will be disabled
    use_acl: true

    # List of super user on this kafka cluster
    super_users:
      - '[[ .instance ]]-jikkou'

    # Kafka brokers settings
    settings:
      log.retention.hours: 168
      compression.type: zstd
      zookeeper.connection.timeout.ms: 600
      zookeeper.set.acl: true
      message.max.bytes: 1073741824
      auto.create.topics.enable: false
      transactional.id.expiration.ms: 2147483647
      offsets.retention.minutes: 13140

    # Volumes for data persistence
    volumes:
      data:
        type: csi
        source: '[[ .instance ]]-broker-data'
        per_alloc: true
  
  # Jikkou can manage topics and ACL on your cluster
  jikkou:
    # Version of Jikkou to use when building the image
    version: 0.34.0
    # Docker image to use
    image: '[[ .docker.repo ]]jikkou:[[ .kafka.jikkou.version ]]-1'
    # Custom env var to set in the container
    env: {}
    # Resource allocation
    resources:
      cpu: 10
      memory: 256

    vault:
      # Vault policies to attach to the task
      policies:
        - '[[ .instance ]]-jikkou[[ .consul.suffix ]]'

    # Wait for a quorum of brokers to be ready before starting the exporter
    wait_for:
      - service: '[[ .instance ]]-broker[[ .consul.suffix ]]'
        count: '[[ .kafka.broker.count | mul 0.5 | add 1 | math.Floor ]]'

    # List of topics to create, as a YAML string. Eg
    #  topics: |
    #    ---
    #    apiVersion: "core.jikkou.io/v1beta2"
    #    kind: ConfigMap
    #    metadata:
    #      name: 'TopicDefaultConfig'
    #    data:
    #      min.insync.replicas: 2
    #      cleanup.policy: delete
    #      compression.type: zstd
    #      retention.ms: 604800000
    #   ---
    #   apiVersion: kafka.jikkou.io/v1beta2
    #   kind: KafkaTopicList
    #   items:
    #     - metadata:
    #         name: my_topic
    #       spec:
    #         partitions: 12
    #         replicas: 3
    #         configMapRefs: [ TopicDefaultConfig ]
    topics: ""

    # List of ACL, as a YAML string. Eg
    # acls: |
    #   ---
    #    apiVersion: kafka.jikkou.io/v1beta2
    #    kind: KafkaPrincipalRole
    #    metadata:
    #      name: foobar-consumer
    #    spec:
    #      acls:
    #        - type: ALLOW
    #          operations: [READ, DESCRIBE]
    #          resource:
    #            type: TOPIC
    #            pattern: {{ $topic }}
    #            patternType: LITERAL
    #          host: '*'
    #        - type: ALLOW
    #          operations: [READ]
    #          resource:
    #            type: GROUP
    #            pattern: {{ $topic }}
    #            patternType: LITERAL
    #          host: '*'
    acls: ""

  # Prometheus exporter
  exporter:
    # Version of the exporter to use when building the image
    version: 1.7.0

    # Docker image to use
    image: '[[ .docker.repo ]]kafka-exporter:[[ .kafka.exporter.version ]]-1'

    # Custom env var to set in the container
    env: {}

    # Wait for a quorum of brokers to be ready before starting the exporter
    wait_for:
      - service: '[[ .instance ]]-broker[[ .consul.suffix ]]'
        count: '[[ .kafka.broker.count | mul 0.5 | add 1 | math.Floor ]]'

    vault:
      # Vault policies to attach to the task
      policies:
        - '[[ .instance ]]-exporter[[ .consul.suffix ]]'
        - metrics[[ .consul.suffix ]]

    # Resource allocation
    resources:
      cpu: 10
      memory: 50