New bundle to create a mongo ReplicaSet
This commit is contained in:
parent
9000767962
commit
bba12b9c85
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
|
||||
dependencies:
|
||||
- url: ../common.git
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
[[- $c := merge .mongo . ]]
|
||||
|
||||
[[ template "common/vault.mkpki.sh" $c ]]
|
||||
|
||||
# Role for mongod
|
||||
vault write [[ $c.vault.pki.path ]]/roles/mongod \
|
||||
allowed_domains="[[ .instance ]][[ .consul.suffix ]],[[ .instance ]].service.[[ .consul.domain ]],addr.[[ .consul.datacenter ]].[[ .consul.domain ]]" \
|
||||
allow_bare_domains=true \
|
||||
allow_subdomains=true \
|
||||
allow_localhost=true \
|
||||
allow_ip_sans=true \
|
||||
server_flag=true \
|
||||
client_flag=true \
|
||||
allow_wildcard_certificates=false \
|
||||
max_ttl=720h \
|
||||
ou=[[ $c.vault.pki.ou ]]
|
||||
|
||||
# This one is for root user certificates
|
||||
vault write [[ $c.vault.pki.path ]]/roles/mongo-root \
|
||||
allowed_domains="mongo.root" \
|
||||
allow_bare_domains=true \
|
||||
allow_subdomains=false \
|
||||
allow_localhost=false \
|
||||
allow_ip_sans=false \
|
||||
server_flag=false \
|
||||
client_flag=true \
|
||||
allow_wildcard_certificates=false \
|
||||
max_ttl=24h \
|
||||
ou=[[ $c.vault.pki.ou ]]
|
||||
|
||||
[[- if .prometheus.enabled ]]
|
||||
|
||||
vault write [[ $c.vault.pki.path ]]/roles/mongo-monitor \
|
||||
allowed_domains="mongo.monitor" \
|
||||
allow_bare_domains=true \
|
||||
allow_subdomains=false \
|
||||
allow_localhost=false \
|
||||
allow_ip_sans=false \
|
||||
server_flag=false \
|
||||
client_flag=true \
|
||||
allow_wildcard_certificates=false \
|
||||
max_ttl=768h \
|
||||
ou=[[ $c.vault.pki.ou ]]-Client
|
||||
[[- end ]]
|
||||
|
||||
# The role for backups
|
||||
vault write [[ $c.vault.pki.path ]]/roles/mongo-backup \
|
||||
allowed_domains="mongo.backup" \
|
||||
allow_bare_domains=true \
|
||||
allow_subdomains=false \
|
||||
allow_localhost=false \
|
||||
allow_ip_sans=false \
|
||||
server_flag=false \
|
||||
client_flag=true \
|
||||
allow_wildcard_certificates=false \
|
||||
max_ttl=12h \
|
||||
ou=[[ $c.vault.pki.ou ]]-Client
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
job "[[ .instance ]]" {
|
||||
|
||||
[[ template "common/job_start" . ]]
|
||||
|
||||
group "mongo" {
|
||||
|
||||
[[- $c := merge .mongo . ]]
|
||||
|
||||
count = [[ $c.count ]]
|
||||
shutdown_delay = "10s"
|
||||
|
||||
constraint {
|
||||
operator = "distinct_hosts"
|
||||
value = "true"
|
||||
}
|
||||
|
||||
network {
|
||||
mode = "bridge"
|
||||
port "mongo" {}
|
||||
[[- if $c.prometheus.enabled ]]
|
||||
port "metrics" {}
|
||||
[[- end ]]
|
||||
}
|
||||
|
||||
[[ template "common/volumes" $c ]]
|
||||
|
||||
# The main service client will use to locate mongo servers
|
||||
service {
|
||||
name = "[[ .instance ]][[ .consul.suffix ]]"
|
||||
port = "mongo"
|
||||
|
||||
[[ template "common/metrics_meta" $c ]]
|
||||
|
||||
check {
|
||||
type = "script"
|
||||
command = "sh"
|
||||
args = ["-c", "/local/bin/mongo --eval 'db.runCommand(\"ping\").ok'"]
|
||||
interval = "15s"
|
||||
timeout = "5s"
|
||||
task = "mongod"
|
||||
}
|
||||
|
||||
tags = [
|
||||
"mongo-${NOMAD_ALLOC_INDEX}"
|
||||
]
|
||||
}
|
||||
|
||||
# This service is just used by the different nodes to find themself and configure the replica set
|
||||
service {
|
||||
name = "[[ .instance ]]-rs[[ .consul.suffix ]]"
|
||||
port = "mongo"
|
||||
meta {
|
||||
alloc = "${NOMAD_ALLOC_INDEX}"
|
||||
}
|
||||
tags = [
|
||||
"mongo-${NOMAD_ALLOC_INDEX}"
|
||||
]
|
||||
}
|
||||
|
||||
# This task will init the cluster on first boot
|
||||
task "mongo-init" {
|
||||
driver = "docker"
|
||||
|
||||
lifecycle {
|
||||
hook = "poststart"
|
||||
}
|
||||
|
||||
config {
|
||||
image = "[[ $c.image ]]"
|
||||
command = "/usr/local/bin/rs_config.sh"
|
||||
pids_limit = 100
|
||||
readonly_rootfs = true
|
||||
# Mount a few files from the main task
|
||||
# Just to reduce the verbosity of templates {} sections
|
||||
volumes = [
|
||||
"../mongod/secrets/mongo.bundle.pem:/secrets/mongo.bundle.pem:ro",
|
||||
"../mongod/secrets/mongo.ca.pem:/secrets/mongo.ca.pem:ro",
|
||||
"../mongod/local/bin/mongosh:/usr/local/bin/mongosh:ro",
|
||||
"../mongod/local/bin/rs_config.sh:/usr/local/bin/rs_config.sh:ro"
|
||||
]
|
||||
[[ template "common/tmpfs" "/tmp" ]]
|
||||
}
|
||||
|
||||
# As the task exits after its done, only reserve little memory
|
||||
# but allow it to use up to 256MB
|
||||
resources {
|
||||
cpu = 20
|
||||
memory = 20
|
||||
memory_max = 256
|
||||
}
|
||||
}
|
||||
|
||||
[[ template "common/task.metrics_proxy" $c ]]
|
||||
|
||||
[[- if $c.prometheus.enabled ]]
|
||||
task "exporter" {
|
||||
|
||||
[[- $e := merge $c.exporter $c ]]
|
||||
|
||||
driver = "[[ $e.nomad.driver ]]"
|
||||
user = "9216"
|
||||
|
||||
config {
|
||||
image = "[[ $e.image ]]"
|
||||
args = [
|
||||
"--mongodb.uri=mongodb://127.0.0.1:${NOMAD_ALLOC_PORT_mongo}/%24external?replicaSet=[[ .mongo.replica_set ]]&authMechanism=MONGODB-X509&tls=true&tlsCertificateKeyFile=%2Fsecrets%2Fmongo.bundle.pem&tlsCAFile=%2Fsecrets%2Fmongo.ca.pem",
|
||||
"--web.listen-address=127.0.0.1:9216",
|
||||
"--collect-all"
|
||||
]
|
||||
pids_limit = 100
|
||||
readonly_rootfs = true
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
hook = "poststart"
|
||||
sidecar = true
|
||||
}
|
||||
|
||||
[[ template "common/vault.policies" $e ]]
|
||||
|
||||
# Get a certificate with monitoring capabilities
|
||||
template {
|
||||
data = <<_EOT
|
||||
{{ with pkiCert "[[ $e.vault.pki.path ]]/issue/mongo-monitor"
|
||||
"common_name=mongo.monitor"
|
||||
"ttl=168h" }}
|
||||
{{ .Cert }}
|
||||
{{ .Key }}
|
||||
{{ end }}
|
||||
_EOT
|
||||
destination = "secrets/mongo.bundle.pem"
|
||||
uid = 109216
|
||||
gid = 109216
|
||||
perms = "0640"
|
||||
}
|
||||
|
||||
# CA cert used to validate remote (mongo servers') certificate. Both root and intermediate
|
||||
template {
|
||||
data = <<_EOT
|
||||
{{ with secret "[[ $e.vault.pki.path ]]/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
|
||||
_EOT
|
||||
destination = "secrets/mongo.ca.pem"
|
||||
}
|
||||
|
||||
[[ template "common/resources" $c.exporter ]]
|
||||
}
|
||||
[[- end ]]
|
||||
|
||||
task "mongod" {
|
||||
|
||||
driver = "[[ $c.nomad.driver ]]"
|
||||
leader = true
|
||||
|
||||
# Give mongod some time to shutdown
|
||||
kill_timeout = "60s"
|
||||
|
||||
config {
|
||||
image = "[[ $c.image ]]"
|
||||
command = "mongod"
|
||||
args = ["--config", "/local/mongod.conf"]
|
||||
pids_limit = 100
|
||||
readonly_rootfs = true
|
||||
volumes = [
|
||||
"local/bin/mongosh:/usr/local/bin/mongosh:ro"
|
||||
]
|
||||
[[ template "common/tmpfs" "/tmp" ]]
|
||||
}
|
||||
|
||||
[[ template "common/vault.policies" $c ]]
|
||||
|
||||
template {
|
||||
data = <<_EOT
|
||||
[[- if isKind "map" $c.config ]]
|
||||
[[ merge $c.config (tmpl.Exec "mongodb/mongod.conf" $c | yaml) | toYAML ]]
|
||||
[[- else if isKind "string" $c.config ]]
|
||||
[[ merge ($c.config | yaml) (tmpl.Exec "mongodb/mongod.conf" $c | yaml) | toYAML ]]
|
||||
[[- else ]]
|
||||
[[ template "mongodb/mongod.conf" $c ]]
|
||||
[[- end ]]
|
||||
_EOT
|
||||
destination = "local/mongod.conf"
|
||||
}
|
||||
|
||||
# This is the certificate used by the mongod process
|
||||
# Note : we ask for a cert valid for the host IP address, and also for the XXXXX.addr.dc1.consul address
|
||||
# as it's what is published in the SRV record. The XXXX is the hex representation of the IP.
|
||||
# Something like {{ range (env "NOMAD_IP_mongo" | split ".") }}{{ printf "%02x" (. | parseInt) }}{{ end }}
|
||||
# would be cleaner, but can't find how to use it in this context
|
||||
# Also : each instance will have a slightly differnt ttl so all cert will not expire at the same time
|
||||
template {
|
||||
data = <<_EOT
|
||||
{{- with pkiCert "[[ $c.vault.pki.path ]]/issue/mongod"
|
||||
(printf "common_name=mongo-%s.[[ .instance ]][[ .consul.suffix ]].service.[[ .consul.domain ]]" (env "NOMAD_ALLOC_INDEX"))
|
||||
(
|
||||
printf "alt_names=%02x%02x%02x%02x.addr.dc1.[[ .consul.domain ]]"
|
||||
(env "NOMAD_HOST_IP_mongo" | regexReplaceAll "^(\\d+)\\..*" "$1" | parseInt)
|
||||
(env "NOMAD_HOST_IP_mongo" | regexReplaceAll "^\\d+\\.(\\d+)\\..*" "$1" | parseInt)
|
||||
(env "NOMAD_HOST_IP_mongo" | regexReplaceAll "^\\d+\\.\\d+\\.(\\d+)\\..*" "$1" | parseInt)
|
||||
(env "NOMAD_HOST_IP_mongo" | regexReplaceAll "^\\d+\\.\\d+\\.\\d+\\.(\\d+)$" "$1" | parseInt)
|
||||
)
|
||||
(printf "ip_sans=%s,127.0.0.1" (env "NOMAD_HOST_IP_mongo"))
|
||||
(printf "ttl=%dh" (env "NOMAD_ALLOC_INDEX" | parseInt | multiply 8 | add 600)) }}
|
||||
{{ .Cert }}
|
||||
{{ .Key }}
|
||||
{{- end }}
|
||||
_EOT
|
||||
destination = "secrets/mongo.bundle.pem"
|
||||
uid = 100999
|
||||
gid = 100999
|
||||
perms = "0640"
|
||||
change_mode = "script"
|
||||
change_script {
|
||||
command = "/local/bin/rotate-cert.sh"
|
||||
# If cert rotation fails, kill the task and restart it
|
||||
fail_on_error = true
|
||||
}
|
||||
}
|
||||
|
||||
# CA cert used to validate remote certs
|
||||
template {
|
||||
data = <<_EOT
|
||||
{{- with secret "[[ $c.vault.pki.path ]]/cert/ca_chain" }}
|
||||
{{ .Data.ca_chain }}
|
||||
{{- end }}
|
||||
_EOT
|
||||
destination = "secrets/mongo.ca.pem"
|
||||
change_mode = "script"
|
||||
change_script {
|
||||
command = "/local/bin/rotate-cert.sh"
|
||||
# If cert rotation fails, kill the task and restart it
|
||||
fail_on_error = true
|
||||
}
|
||||
}
|
||||
|
||||
# A mongosh wrapper which will automatically use the x509 root cert
|
||||
template {
|
||||
data = <<_EOT
|
||||
[[ template "mongodb/mongosh" $c ]]
|
||||
_EOT
|
||||
destination = "local/bin/mongosh"
|
||||
uid = 100000
|
||||
gid = 100000
|
||||
perms = "0755"
|
||||
}
|
||||
|
||||
# Same for mongo.
|
||||
# Note : mongo tool is deprecated and replaced by mongosh. But mongosh has huge performance
|
||||
# issues, making it unusable for health checks for example
|
||||
template {
|
||||
data = <<_EOT
|
||||
[[ template "mongodb/mongo" $c ]]
|
||||
_EOT
|
||||
destination = "local/bin/mongo"
|
||||
uid = 100000
|
||||
gid = 100000
|
||||
perms = "0755"
|
||||
}
|
||||
|
||||
# A script to dynamically reconfigure members' address when mongo instances restart
|
||||
template {
|
||||
data = <<_EOT
|
||||
[[ template "mongodb/rs_config.sh" $c ]]
|
||||
_EOT
|
||||
destination = "local/bin/rs_config.sh"
|
||||
uid = 100000
|
||||
gid = 100000
|
||||
perms = "0755"
|
||||
change_mode = "script"
|
||||
change_script {
|
||||
command = "/local/bin/rs_config.sh"
|
||||
# Will fail when mongod is shuting down. We don't want Nomad to kill the task
|
||||
fail_on_error = false
|
||||
}
|
||||
}
|
||||
|
||||
# A script to rotate certificates
|
||||
template {
|
||||
data = <<_EOT
|
||||
[[ template "mongodb/rotate-cert.sh" $c ]]
|
||||
_EOT
|
||||
destination = "local/bin/rotate-cert.sh"
|
||||
uid = 100000
|
||||
gid = 100000
|
||||
perms = "0755"
|
||||
}
|
||||
|
||||
# Persistent data
|
||||
volume_mount {
|
||||
volume = "mongo"
|
||||
destination = "/data"
|
||||
}
|
||||
|
||||
[[ template "common/resources" $c ]]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/sh
|
||||
|
||||
# mongo wrapper which just add x509 auth args
|
||||
|
||||
set -eu
|
||||
|
||||
/bin/mongo \
|
||||
--port ${NOMAD_PORT_mongo} \
|
||||
--tls \
|
||||
--tlsCertificateKeyFile /secrets/mongo.bundle.pem \
|
||||
--tlsCAFile /secrets/mongo.ca.pem \
|
||||
--authenticationDatabase '$external' \
|
||||
--authenticationMechanism MONGODB-X509 \
|
||||
--quiet \
|
||||
"$@"
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
net:
|
||||
tls:
|
||||
mode: requireTLS
|
||||
certificateKeyFile: /secrets/mongo.bundle.pem
|
||||
CAFile: /secrets/mongo.ca.pem
|
||||
port: '{{ env "NOMAD_PORT_mongo" }}'
|
||||
bindIpAll: true
|
||||
|
||||
security:
|
||||
clusterAuthMode: x509
|
||||
authorization: enabled
|
||||
|
||||
setParameter:
|
||||
disableSplitHorizonIPCheck: true
|
||||
tlsX509ExpirationWarningThresholdDays: 1
|
||||
watchdogPeriodSeconds: 120
|
||||
tcmallocReleaseRate: 5.0
|
||||
maxSessions: 1000
|
||||
shutdownTimeoutMillisForSignaledShutdown: 3000
|
||||
|
||||
storage:
|
||||
dbPath: /data/db
|
||||
directoryPerDB: true
|
||||
wiredTiger:
|
||||
engineConfig:
|
||||
directoryForIndexes: true
|
||||
journalCompressor: zstd
|
||||
collectionConfig:
|
||||
blockCompressor: zstd
|
||||
|
||||
replication:
|
||||
replSetName: [[ .mongo.replica_set ]]
|
||||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/sh
|
||||
|
||||
# mongosh wrapper which just add x509 auth args
|
||||
|
||||
set -eu
|
||||
|
||||
/bin/mongosh \
|
||||
--port ${NOMAD_PORT_mongo} \
|
||||
--tls \
|
||||
--tlsCertificateKeyFile /secrets/mongo.bundle.pem \
|
||||
--tlsCAFile /secrets/mongo.ca.pem \
|
||||
--authenticationDatabase '$external' \
|
||||
--authenticationMechanism MONGODB-X509 \
|
||||
--quiet \
|
||||
"$@"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -eux
|
||||
/local/bin/mongosh --tlsAllowInvalidCertificates --eval <<_EOF
|
||||
db.adminCommand( { rotateCertificates: 1, message: "Rotating certificates" } );
|
||||
_EOF
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
sleep 2
|
||||
if mongosh --eval "rs.conf()" >/dev/null 1>&1; then
|
||||
echo "ReplicaSet already configured, reconfiguring members' address"
|
||||
mongosh --eval << '_EOF'
|
||||
rs_conf = rs.conf();
|
||||
|
||||
{{- range $index, $instance := service "[[ .instance ]][[ .consul.suffix ]]" }}
|
||||
rs_conf.members[{{ index $instance.ServiceMeta "alloc" }}] = {
|
||||
_id : {{ index $instance.ServiceMeta "alloc" }},
|
||||
host : "{{ $instance.Address }}:{{ $instance.Port }}"
|
||||
};
|
||||
{{- end }}
|
||||
rs.reconfig(rs_conf, { "force": true });
|
||||
_EOF
|
||||
|
||||
elif [ "${NOMAD_ALLOC_INDEX}" != "0" ]; then
|
||||
echo "Initialization will only run on NOMAD_ALLOC_INDEX 0, exiting"
|
||||
exit 0
|
||||
|
||||
else
|
||||
echo "Initializing replica set"
|
||||
sleep 60
|
||||
|
||||
mongosh \
|
||||
--eval << '_EOF'
|
||||
rs_conf = {
|
||||
_id : "[[ .mongo.replica_set ]]",
|
||||
members : []
|
||||
};
|
||||
{{- range $index, $instance := service "[[ .instance ]]-rs[[ .consul.suffix ]]" }}
|
||||
rs_conf.members[{{ index $instance.ServiceMeta "alloc" }}] = {
|
||||
_id : {{ index $instance.ServiceMeta "alloc" }},
|
||||
host : "{{ $instance.Address }}:{{ $instance.Port }}"
|
||||
};
|
||||
{{- end }}
|
||||
rs.initiate(rs_conf);
|
||||
_EOF
|
||||
|
||||
echo "Creating root user"
|
||||
mongosh \
|
||||
--eval << '_EOF'
|
||||
db.getSiblingDB("$external").createUser({
|
||||
user: "CN=mongo.root,OU=[[ .vault.pki.ou ]]",
|
||||
roles: [{ role: "root", db: "admin" }]
|
||||
});
|
||||
_EOF
|
||||
|
||||
fi
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
---
|
||||
|
||||
# Name of the instance
|
||||
instance: mongodb
|
||||
|
||||
# Vault settings for the PKI
|
||||
vault:
|
||||
pki:
|
||||
ou: MongoDB
|
||||
|
||||
mongo:
|
||||
|
||||
# Docker image to use
|
||||
image: '[[ .docker.repo ]][[ .docker.base_images.mongo50.image ]]'
|
||||
|
||||
# Number of instances to run
|
||||
count: 3
|
||||
|
||||
# Custom env var to set in the containers
|
||||
env: {}
|
||||
|
||||
# Resource allocation for each mongo node
|
||||
resources:
|
||||
cpu: 200
|
||||
memory: 768
|
||||
|
||||
# Vault settings
|
||||
vault:
|
||||
# List of policies to attach to the containers
|
||||
policies:
|
||||
- '[[ .instance ]]-mongod[[ .consul.suffix ]]'
|
||||
|
||||
# Random secrets to generate
|
||||
rand_secrets:
|
||||
fields:
|
||||
- root_pwd
|
||||
|
||||
# Custom mongod.conf fragment which will be merged with the default one
|
||||
# Can be either a yaml object or a string (if using consul-template is needed)
|
||||
config: {}
|
||||
|
||||
# Volume for data persistence
|
||||
volumes:
|
||||
mongo:
|
||||
source: mongo-data
|
||||
type: csi
|
||||
per_alloc: true
|
||||
|
||||
# Name of the replica set
|
||||
replica_set: rs0
|
||||
|
||||
# Prometheus exporter
|
||||
exporter:
|
||||
version: 0.40.0
|
||||
image: percona/mongodb_exporter:[[ .mongo.exporter.version ]]
|
||||
resources:
|
||||
cpu: 10
|
||||
memory: 50
|
||||
|
||||
prometheus:
|
||||
# URL where prometheus metrics are exposed (from inside the container PoV)
|
||||
metrics_url: http://127.0.0.1:9216/metrics
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
[[- $c := merge .mongo . ]]
|
||||
|
||||
# Read secrets from the KV store
|
||||
path "[[ $c.vault.root ]]kv/data/service/[[ .instance ]]" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
# Issue cert for mongod
|
||||
path "[[ $c.vault.pki.path ]]/issue/mongod" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
|
||||
[[- if .prometheus.enabled ]]
|
||||
# Issue client cert for the exporter
|
||||
path "[[ $c.vault.pki.path ]]/issue/mongo-monitor" {
|
||||
capabilities = ["update"]
|
||||
}
|
||||
[[- end ]]
|
||||
Loading…
Reference in New Issue