200 lines
5.3 KiB
YAML
200 lines
5.3 KiB
YAML
---
|
|
|
|
# Name of this instance
|
|
# Only usefull if you intent to run several instances on your cluster (in the same namespace)
|
|
instance: kafka
|
|
|
|
vault:
|
|
# PKI settings
|
|
pki:
|
|
path: "[[ .vault.root ]]pki/kafka"
|
|
ou: Kafka Cluster ([[ .instance ]])
|
|
|
|
kafka:
|
|
broker:
|
|
# Number of brokers to run
|
|
count: 3
|
|
|
|
# Version of Kafka to use to build the image
|
|
version: 3.7.0
|
|
|
|
# Docker image to use
|
|
image: '[[ .docker.repo ]]kafka-broker:[[ .kafka.broker.version ]]-4'
|
|
|
|
# Custom env var to set in the container
|
|
env: {}
|
|
|
|
# Resource allocation for each broker
|
|
resources:
|
|
cpu: 100
|
|
memory: 1024
|
|
|
|
# Prometheus settings
|
|
prometheus:
|
|
enabled: '[[ .prometheus.available ]]'
|
|
metrics_url: http://127.0.0.1:9404/metrics
|
|
|
|
# You can set static ports for both client and broker port. If not set, ports will be allocated dynamicaly by Nomad
|
|
# static_client_port: 9092
|
|
# static_broker_port: 9095
|
|
# Ensure all broker runs on distinct hosts
|
|
nomad:
|
|
constraints:
|
|
- operator: distinct_hosts
|
|
value: true
|
|
|
|
# Consul settings
|
|
consul:
|
|
meta:
|
|
# Push the name of the broker as a meta of the service
|
|
broker: 'broker-${NOMAD_ALLOC_INDEX}.[[ .instance ]]-broker[[ .consul.suffix ]].service.[[ .consul.domain ]]'
|
|
# Service mesh
|
|
connect:
|
|
upstreams:
|
|
# Default is to contact a zookeeper cluster through the mesh
|
|
- destination_name: zookeeper[[ .consul.suffix ]]
|
|
local_bind_port: 2181
|
|
|
|
# Vault settings
|
|
vault:
|
|
# Policies to attach to the task
|
|
policies:
|
|
- '[[ .instance ]]-broker[[ .consul.suffix ]]'
|
|
|
|
# Wait for ZooKeeper to be available before starting brokers
|
|
wait_for:
|
|
- service: zookeeper[[ .consul.suffix ]]
|
|
count: 2
|
|
|
|
# ZpooKeeper settings
|
|
zookeeper:
|
|
servers:
|
|
- 127.0.0.1:2181
|
|
# path: /kafka
|
|
user: ""
|
|
password: ""
|
|
|
|
# If set to false, ACL will be disabled
|
|
use_acl: true
|
|
|
|
# List of super user on this kafka cluster
|
|
super_users:
|
|
- '[[ .instance ]]-jikkou'
|
|
|
|
# Kafka brokers settings
|
|
settings:
|
|
log.retention.hours: 168
|
|
compression.type: zstd
|
|
zookeeper.connection.timeout.ms: 600
|
|
zookeeper.set.acl: true
|
|
message.max.bytes: 1073741824
|
|
auto.create.topics.enable: false
|
|
transactional.id.expiration.ms: 2147483647
|
|
offsets.retention.minutes: 13140
|
|
|
|
# Volumes for data persistence
|
|
volumes:
|
|
data:
|
|
type: csi
|
|
source: '[[ .instance ]]-broker-data'
|
|
per_alloc: true
|
|
|
|
# Jikkou can manage topics and ACL on your cluster
|
|
jikkou:
|
|
# Version of Jikkou to use when building the image
|
|
version: 0.34.0
|
|
# Docker image to use
|
|
image: '[[ .docker.repo ]]jikkou:[[ .kafka.jikkou.version ]]-1'
|
|
# Custom env var to set in the container
|
|
env: {}
|
|
# Resource allocation
|
|
resources:
|
|
cpu: 10
|
|
memory: 256
|
|
|
|
vault:
|
|
# Vault policies to attach to the task
|
|
policies:
|
|
- '[[ .instance ]]-jikkou[[ .consul.suffix ]]'
|
|
|
|
# Wait for a quorum of brokers to be ready before starting the exporter
|
|
wait_for:
|
|
- service: '[[ .instance ]]-broker[[ .consul.suffix ]]'
|
|
count: '[[ .kafka.broker.count | mul 0.5 | add 1 | math.Floor ]]'
|
|
|
|
# List of topics to create, as a YAML string. Eg
|
|
# topics: |
|
|
# ---
|
|
# apiVersion: "core.jikkou.io/v1beta2"
|
|
# kind: ConfigMap
|
|
# metadata:
|
|
# name: 'TopicDefaultConfig'
|
|
# data:
|
|
# min.insync.replicas: 2
|
|
# cleanup.policy: delete
|
|
# compression.type: zstd
|
|
# retention.ms: 604800000
|
|
# ---
|
|
# apiVersion: kafka.jikkou.io/v1beta2
|
|
# kind: KafkaTopicList
|
|
# items:
|
|
# - metadata:
|
|
# name: my_topic
|
|
# spec:
|
|
# partitions: 12
|
|
# replicas: 3
|
|
# configMapRefs: [ TopicDefaultConfig ]
|
|
topics: ""
|
|
|
|
# List of ACL, as a YAML string. Eg
|
|
# acls: |
|
|
# ---
|
|
# apiVersion: kafka.jikkou.io/v1beta2
|
|
# kind: KafkaPrincipalRole
|
|
# metadata:
|
|
# name: foobar-consumer
|
|
# spec:
|
|
# acls:
|
|
# - type: ALLOW
|
|
# operations: [READ, DESCRIBE]
|
|
# resource:
|
|
# type: TOPIC
|
|
# pattern: {{ $topic }}
|
|
# patternType: LITERAL
|
|
# host: '*'
|
|
# - type: ALLOW
|
|
# operations: [READ]
|
|
# resource:
|
|
# type: GROUP
|
|
# pattern: {{ $topic }}
|
|
# patternType: LITERAL
|
|
# host: '*'
|
|
acls: ""
|
|
|
|
# Prometheus exporter
|
|
exporter:
|
|
# Version of the exporter to use when building the image
|
|
version: 1.7.0
|
|
|
|
# Docker image to use
|
|
image: '[[ .docker.repo ]]kafka-exporter:[[ .kafka.exporter.version ]]-1'
|
|
|
|
# Custom env var to set in the container
|
|
env: {}
|
|
|
|
# Wait for a quorum of brokers to be ready before starting the exporter
|
|
wait_for:
|
|
- service: '[[ .instance ]]-broker[[ .consul.suffix ]]'
|
|
count: '[[ .kafka.broker.count | mul 0.5 | add 1 | math.Floor ]]'
|
|
|
|
vault:
|
|
# Vault policies to attach to the task
|
|
policies:
|
|
- '[[ .instance ]]-exporter[[ .consul.suffix ]]'
|
|
- metrics[[ .consul.suffix ]]
|
|
|
|
# Resource allocation
|
|
resources:
|
|
cpu: 10
|
|
memory: 50
|