This commit is contained in:
Daniel Berteaud 2023-12-21 23:15:47 +01:00
parent 4e36c99901
commit 00561f2ccf
19 changed files with 75 additions and 74 deletions

View File

@ -1,8 +1,8 @@
Kind = "service-intentions"
Name = "[[ .pg.instance ]][[ .consul.suffix ]]"
Name = "[[ .instance ]][[ .consul.suffix ]]"
Sources = [
{
Name = "[[ .pg.instance ]]-manage[[ .consul.suffix ]]"
Name = "[[ .instance ]]-manage[[ .consul.suffix ]]"
Action = "allow"
},
{

View File

@ -1,6 +1,6 @@
Kind = "service-resolver"
Name = "[[ .pg.instance ]]-master[[ .consul.suffix ]]"
Name = "[[ .instance ]]-master[[ .consul.suffix ]]"
Redirect {
Service = "[[ .pg.instance ]][[ .consul.suffix ]]"
Service = "[[ .instance ]][[ .consul.suffix ]]"
ServiceSubset = "master"
}

View File

@ -1,6 +1,6 @@
Kind = "service-resolver"
Name = "[[ .pg.instance ]]-replica[[ .consul.suffix ]]"
Name = "[[ .instance ]]-replica[[ .consul.suffix ]]"
Redirect {
Service = "[[ .pg.instance ]][[ .consul.suffix ]]"
Service = "[[ .instance ]][[ .consul.suffix ]]"
ServiceSubset = "replica"
}

View File

@ -1,5 +1,5 @@
Kind = "service-resolver"
Name = "[[ .pg.instance ]][[ .consul.suffix ]]"
Name = "[[ .instance ]][[ .consul.suffix ]]"
DefaultSubset = "master"
Subsets = {
"master" = {

View File

@ -1,13 +1,13 @@
node_prefix "" {
policy = "read"
}
service "[[ .pg.instance ]]" {
service "[[ .instance ]]" {
policy = "write"
}
service "[[ .pg.instance ]]-sidecar-proxy" {
service "[[ .instance ]]-sidecar-proxy" {
policy = "write"
}
key_prefix "[[ .consul.prefix ]]service/[[ .pg.instance ]]" {
key_prefix "[[ .consul.prefix ]]service/[[ .instance ]]" {
policy = "write"
}
session_prefix "" {

View File

@ -1,4 +1,4 @@
#!/bin/sh
# vim: syntax=sh
vault write [[ .vault.prefix ]]consul/roles/[[ .pg.instance ]] ttl=720h max_ttl=720h consul_policies="[[ .pg.instance ]]"
vault write [[ .vault.prefix ]]consul/roles/[[ .instance ]] ttl=720h max_ttl=720h consul_policies="[[ .instance ]]"

View File

@ -1,11 +1,11 @@
#!/bin/sh
for USER in pg monitor replicator rewind api vault_initial; do
vault kv get -field ${USER}_pwd [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] > /dev/null 2>&1
vault kv get -field ${USER}_pwd [[ .vault.prefix ]]kv/service/[[ .instance ]] > /dev/null 2>&1
RES=$?
if [ "${RES}" = "1" ]; then
vault kv patch [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
vault kv patch [[ .vault.prefix ]]kv/service/[[ .instance ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
elif [ "${RES}" = "2" ]; then
vault kv put [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
vault kv put [[ .vault.prefix ]]kv/service/[[ .instance ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
fi
done

View File

@ -4,14 +4,14 @@
[[ template "common/vault.mkpki.sh.tpl" dict
"ctx" .
"pki" (dict
"name" .pg.instance
"name" .instance
"ou" "PostgreSQL"
"issuer" "pki/root"
)
]]
vault write [[ .vault.prefix ]]pki/[[ .pg.instance ]]/roles/postgres-server \
allowed_domains="[[ .pg.instance ]].service.[[ .consul.domain ]]" \
vault write [[ .vault.prefix ]]pki/[[ .instance ]]/roles/postgres-server \
allowed_domains="[[ .instance ]].service.[[ .consul.domain ]]" \
allow_bare_domains=true \
allow_subdomains=true \
allow_localhost=false \

View File

@ -8,20 +8,20 @@ else
echo "Database secret already enabled at [[ .vault.prefix ]]database"
fi
if [ "$(vault list -format json [[ .vault.prefix ]]database/config | jq '.[] | test("^[[ .pg.instance ]]$")')" != "true" ]; then
echo "Configuring database plugin [[ .vault.prefix ]]database/config/[[ .pg.instance ]]"
vault write [[ .vault.prefix ]]database/config/[[ .pg.instance ]] \
if [ "$(vault list -format json [[ .vault.prefix ]]database/config | jq '.[] | test("^[[ .instance ]]$")')" != "true" ]; then
echo "Configuring database plugin [[ .vault.prefix ]]database/config/[[ .instance ]]"
vault write [[ .vault.prefix ]]database/config/[[ .instance ]] \
plugin_name="postgresql-database-plugin" \
connection_url="postgresql://{{username}}:{{password}}@[[ (urlParse .pg.server.public_url).Host ]]/postgres" \
allowed_roles="*" \
username=vault \
password="$(vault kv get -field vault_initial_pwd [[ .vault.prefix ]]kv/service/[[ .pg.instance ]])" \
password="$(vault kv get -field vault_initial_pwd [[ .vault.prefix ]]kv/service/[[ .instance ]])" \
password_authentication=scram-sha-256 \
disable_escaping=true
echo "Rotating root password"
vault write -force [[ .vault.prefix ]]database/rotate-root/[[ .pg.instance ]]
vault write -force [[ .vault.prefix ]]database/rotate-root/[[ .instance ]]
else
echo "Database plugin already configured for [[ .vault.prefix ]]database/config/[[ .pg.instance ]]"
echo "Database plugin already configured for [[ .vault.prefix ]]database/config/[[ .instance ]]"
fi
[[- else ]]
echo "Required .pg.server.public_url is missing"
@ -30,5 +30,5 @@ echo "Required .pg.server.public_url is missing"
echo "Creating dba role in vault"
[[- template "common/vault.mkpgrole.sh.tpl"
dict "ctx" .
"config" (dict "role" (printf "%s-admin" .pg.instance) "pgrole" "dba" "database" .pg.instance)
"config" (dict "role" (printf "%s-admin" .instance) "pgrole" "dba" "database" .instance)
]]

View File

@ -1,6 +1,6 @@
[[ $c := merge .pg.manage . -]]
job "[[ .pg.instance ]]-manage[[ $c.consul.suffix ]]" {
job "[[ .instance ]]-manage[[ $c.consul.suffix ]]" {
[[- if and (has $c "cron") (ne $c.cron "") ]]
type = "service"
@ -25,7 +25,7 @@ job "[[ .pg.instance ]]-manage[[ $c.consul.suffix ]]" {
}
service {
name = "[[ .pg.instance ]]-manage[[ $c.consul.suffix ]]"
name = "[[ .instance ]]-manage[[ $c.consul.suffix ]]"
[[ template "common/connect" $c ]]
}
@ -41,7 +41,7 @@ job "[[ .pg.instance ]]-manage[[ $c.consul.suffix ]]" {
}
vault {
policies = ["[[ .pg.instance ]][[ $c.consul.suffix ]]"]
policies = ["[[ .instance ]][[ $c.consul.suffix ]]"]
env = false
disable_file = true
}
@ -79,7 +79,7 @@ job "[[ .pg.instance ]]-manage[[ $c.consul.suffix ]]" {
PGHOST=localhost
PGPORT=5432
PGUSER=postgres
PGPASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.instance ]]" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
PGPASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
_EOF
destination = "secrets/pg-manage.env"
uid = 100000

View File

@ -1,6 +1,6 @@
[[ $c := merge .pg.server . -]]
job [[ .pg.instance | toJSON ]] {
job [[ .instance | toJSON ]] {
[[ template "common/job_start" $c ]]
@ -42,7 +42,7 @@ job [[ .pg.instance | toJSON ]] {
[[- if not .pg.server.recovery ]]
service {
name = "[[ .pg.instance ]][[ $c.consul.suffix ]]"
name = "[[ .instance ]][[ $c.consul.suffix ]]"
port = 5432
[[- template "common/prometheus_meta" $c ]]
[[- template "common/connect" $c ]]
@ -51,11 +51,11 @@ job [[ .pg.instance | toJSON ]] {
[[- if .pg.server.traefik.enabled ]]
# Note : we don't add traefik.enable=true
# This will be done dynamically only on the current master node using the update_tags.sh script
"[[ $c.traefik.instance ]].tcp.routers.[[ .pg.instance ]][[ $c.consul.suffix ]].rule=HostSNI(`[[ if has .pg.server "public_url" ]][[ (urlParse .pg.server.public_url).Hostname ]][[ else ]]*[[ end ]]`)",
"[[ $c.traefik.instance ]].tcp.routers.[[ .pg.instance ]][[ $c.consul.suffix ]].tls=true",
"[[ $c.traefik.instance ]].tcp.routers.[[ .pg.instance ]][[ $c.consul.suffix ]].entrypoints=[[ join $c.traefik.entrypoints "," ]]",
"[[ $c.traefik.instance ]].tcp.routers.[[ .instance ]][[ $c.consul.suffix ]].rule=HostSNI(`[[ if has .pg.server "public_url" ]][[ (urlParse .pg.server.public_url).Hostname ]][[ else ]]*[[ end ]]`)",
"[[ $c.traefik.instance ]].tcp.routers.[[ .instance ]][[ $c.consul.suffix ]].tls=true",
"[[ $c.traefik.instance ]].tcp.routers.[[ .instance ]][[ $c.consul.suffix ]].entrypoints=[[ join $c.traefik.entrypoints "," ]]",
[[- if gt (len .pg.server.traefik.middlewares) 0 ]]
"[[ $c.traefik.instance ]].tcp.routers.[[ .pg.instance ]][[ $c.consul.suffix ]].middlewares=[[ join $c.traefik.middlewares "," ]]",
"[[ $c.traefik.instance ]].tcp.routers.[[ .instance ]][[ $c.consul.suffix ]].middlewares=[[ join $c.traefik.middlewares "," ]]",
[[- end ]]
[[- end ]]
]
@ -103,7 +103,7 @@ job [[ .pg.instance | toJSON ]] {
[[- if $c.prometheus.enabled ]]
# This service is just used to expose patroni metrics
service {
name = "[[ .pg.instance ]]-patroni[[ $c.consul.suffix ]]"
name = "[[ .instance ]]-patroni[[ $c.consul.suffix ]]"
port = "patroni"
meta {
@ -158,7 +158,7 @@ job [[ .pg.instance | toJSON ]] {
}
vault {
policies = ["[[ .pg.instance ]][[ $c.consul.suffix ]]"]
policies = ["[[ .instance ]][[ $c.consul.suffix ]]"]
env = false
disable_file = true
}
@ -168,9 +168,9 @@ job [[ .pg.instance | toJSON ]] {
template {
data = <<_EOT
# Get a Consul token from vault, so we're able to update the tags in Consul from the containers
CONSUL_HTTP_TOKEN={{ with secret "[[ .vault.prefix ]]consul/creds/[[ .pg.instance ]]" }}{{ .Data.token }}{{ end }}
CONSUL_HTTP_TOKEN={{ with secret "[[ .vault.prefix ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
PATRONICTL_CONFIG_FILE=/secrets/patroni.yml
PGBACKREST_STANZA=[[ .pg.instance ]]
PGBACKREST_STANZA=[[ .instance ]]
_EOT
destination = "secrets/pg.env"
uid = 100000
@ -254,8 +254,8 @@ _EOT
template {
data = <<_EOT
{{ with pkiCert
"[[ .vault.prefix ]]pki/[[ .pg.instance ]]/issue/postgres-server"
"common_name=[[ .pg.instance ]].service.[[ .consul.domain ]]"
"[[ .vault.prefix ]]pki/[[ .instance ]]/issue/postgres-server"
"common_name=[[ .instance ]].service.[[ .consul.domain ]]"
(printf "ip_sans=%s" (env "NOMAD_IP_patroni")) "ttl=72h" }}
{{ .Cert }}
{{ .Key }}
@ -272,7 +272,7 @@ _EOT
# CA certificate chains
template {
data = <<_EOT
{{ with secret "[[ .vault.prefix ]]pki/[[ .pg.instance ]]/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
{{ with secret "[[ .vault.prefix ]]pki/[[ .instance ]]/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
destination = "local/postgres.ca.pem"
change_mode = "signal"
@ -316,7 +316,7 @@ _EOT
}
env {
PGBACKREST_STANZA = [[ .pg.instance | toJSON ]]
PGBACKREST_STANZA = [[ .instance | toJSON ]]
# Use the socket from the shared dir
PGHOST = "/alloc/data/postgres"
}
@ -378,7 +378,7 @@ _EOT
}
vault {
policies = ["[[ .pg.instance ]][[ $c.consul.suffix ]]"]
policies = ["[[ .instance ]][[ $c.consul.suffix ]]"]
env = false
disable_file = true
}

View File

@ -4,8 +4,8 @@ set -euo pipefail
# Initialize random passwords if needed
if ! vault kv list [[ .vault.prefix ]]kv/service 2>/dev/null | grep -q -E '^[[ .pg.instance ]]$'; then
vault kv put [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] \
if ! vault kv list [[ .vault.prefix ]]kv/service 2>/dev/null | grep -q -E '^[[ .instance ]]$'; then
vault kv put [[ .vault.prefix ]]kv/service/[[ .instance ]] \
pg_pwd=$(pwgen -s -n 50 1) \
api_pwd=$(pwgen -s -n 50 1) \
monitor_pwd=$(pwgen -s -n 50 1) \
@ -15,8 +15,8 @@ if ! vault kv list [[ .vault.prefix ]]kv/service 2>/dev/null | grep -q -E '^[[ .
fi
for PWD in pg_pwd api_pwd monitor_pwd replicator_pwd rewind_pwd vault_initial_pwd; do
if ! vault kv get -field ${PWD} [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] >/dev/null 2>&1; then
vault kv patch [[ .vault.prefix ]]kv/service/[[ .pg.instance ]] \
if ! vault kv get -field ${PWD} [[ .vault.prefix ]]kv/service/[[ .instance ]] >/dev/null 2>&1; then
vault kv patch [[ .vault.prefix ]]kv/service/[[ .instance ]] \
${PWD}=$(pwgen -s -n 50 1)
fi
done

View File

@ -1 +1 @@
[[ template "common/mv_conf.sh.tpl" dict "ctx" . "services" (dict "postgres" .pg.instance) ]]
[[ template "common/mv_conf.sh.tpl" dict "ctx" . "services" (dict "postgres" .instance) ]]

View File

@ -3,7 +3,7 @@
set -euo pipefail
# Create roles needed for patroni
{{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.instance ]]" }}
{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}
psql <<'_EOSQL'
ALTER ROLE postgres WITH SUPERUSER LOGIN PASSWORD '{{ .Data.data.pg_pwd }}';
CREATE ROLE replicator WITH LOGIN REPLICATION PASSWORD '{{ .Data.data.replicator_pwd }}';

View File

@ -1,9 +1,9 @@
name: [[ .pg.instance ]]-{{ env "NOMAD_ALLOC_INDEX" }}
scope: [[ .pg.instance ]]
name: [[ .instance ]]-{{ env "NOMAD_ALLOC_INDEX" }}
scope: [[ .instance ]]
consul:
url: http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500
token: {{ with secret "[[ .consul.prefix ]]consul/creds/[[ .pg.instance ]]" }}{{ .Data.token }}{{ end }}
token: {{ with secret "[[ .consul.prefix ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
bootstrap:
dcs:
@ -65,9 +65,9 @@ postgresql:
- hostssl all all 0.0.0.0/0 cert clientcert=verify-full
pg_ident:
- patroni-map [[ .pg.instance ]].service.[[ .consul.domain ]] postgres
- patroni-map [[ .pg.instance ]].service.[[ .consul.domain ]] replicator
- patroni-map [[ .pg.instance ]].service.[[ .consul.domain ]] rewind
- patroni-map [[ .instance ]].service.[[ .consul.domain ]] postgres
- patroni-map [[ .instance ]].service.[[ .consul.domain ]] replicator
- patroni-map [[ .instance ]].service.[[ .consul.domain ]] rewind
parameters:
ssl: on
@ -109,7 +109,7 @@ postgresql:
authentication:
superuser:
username: postgres
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.instance ]]" }}{{ .Data.data.pg_pwd }}{{ end }}'
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd }}{{ end }}'
sslmode: verify-ca
sslrootcert: /local/postgres.ca.pem
@ -136,7 +136,7 @@ restapi:
verify_client: optional
authentication:
username: patroni
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.instance ]]" }}{{ .Data.data.api_pwd }}{{ end }}'
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.api_pwd }}{{ end }}'
ctl:
insecure: False

View File

@ -18,11 +18,11 @@ fi
CURL_OPTS="--connect-timeout 5 --max-time 10 --retry 5 --retry-delay 1 --retry-max-time 40 --retry-connrefused"
# Update tags on the main service
curl ${CURL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/catalog/service/[[ .pg.instance ]] |\
jq --from-file /local/serviceformat.jq --arg role ${NEW_ROLE} --arg mytag [[ .pg.instance ]]-{{ env "NOMAD_ALLOC_INDEX" }} |\
curl ${CURL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/catalog/service/[[ .instance ]] |\
jq --from-file /local/serviceformat.jq --arg role ${NEW_ROLE} --arg mytag [[ .instance ]]-{{ env "NOMAD_ALLOC_INDEX" }} |\
curl ${CORL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" -X PUT -d @- http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/agent/service/register
# Update tags on the sidecar service (connect-proxy)
curl ${CURL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/catalog/service/[[ .pg.instance ]]-sidecar-proxy |\
jq --from-file /local/serviceformat.jq --arg role ${NEW_ROLE} --arg mytag [[ .pg.instance ]]-{{ env "NOMAD_ALLOC_INDEX" }} |\
curl ${CURL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/catalog/service/[[ .instance ]]-sidecar-proxy |\
jq --from-file /local/serviceformat.jq --arg role ${NEW_ROLE} --arg mytag [[ .instance ]]-{{ env "NOMAD_ALLOC_INDEX" }} |\
curl ${CURL_OPTS} -H "X-Consul-Token: ${CONSUL_HTTP_TOKEN}" -X PUT -d @- http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500/v1/agent/service/register

View File

@ -1,5 +1,5 @@
[[ $c := merge .pg.upgrade . -]]
job "[[ .pg.instance ]]-upgrade" {
job "[[ .instance ]]-upgrade" {
[[ template "common/job_start.tpl" $c ]]
@ -23,7 +23,7 @@ job "[[ .pg.instance ]]-upgrade" {
}
task "[[ .pg.instance ]]-upgrade" {
task "[[ .instance ]]-upgrade" {
driver = [[ $c.nomad.driver | toJSON ]]
config {

View File

@ -1,9 +1,10 @@
---
# Name of the job to generate
# Also used to controler service names
instance: postgres
pg:
# Name of the job to generate
# Also used to controler service names
instance: postgres
# Postgres server settings
server:
@ -98,7 +99,7 @@ pg:
# Optional bind DN and password to do the search operation
# If undefined, the search will be done anonymously
#bind_dn: CN=Postgres,OU=Apps,DC=example,DC=org
bind_password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.instance ]]" }}{{ .Data.data.ldap_pwd }}{{ end }}'
bind_password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.ldap_pwd }}{{ end }}'
# Resource allocation for the container
resources:
@ -179,7 +180,7 @@ pg:
# Services to wait before running
wait_for:
- service: 'master.[[ .pg.instance ]]'
- service: 'master.[[ .instance ]]'
# Additional env var
env: {}
@ -188,7 +189,7 @@ pg:
consul:
connect:
upstreams:
- destination_name: "[[ .pg.instance ]]"
- destination_name: "[[ .instance ]]"
local_bind_port: 5432
# List of databases to create (so permissions can be applied)
@ -344,9 +345,9 @@ pg:
# The volumes are connected using per_alloc, so the alloc ID will be appended. Eg postgres-data[0], postgres-data[1] etc.
data:
type: csi
source: '[[ .pg.instance ]]-data'
source: '[[ .instance ]]-data'
# Backup volume (can be used for pgbackrest and dumps)
# Will be opened as multi-node-multi-writer. Can be NFS
backup:
type: csi
source: '[[ .pg.instance ]]-backup'
source: '[[ .instance ]]-backup'

View File

@ -1,14 +1,14 @@
# Read secrets from vault KV
path "[[ .vault.prefix ]]kv/data/service/[[ .pg.instance ]]" {
path "[[ .vault.prefix ]]kv/data/service/[[ .instance ]]" {
capabilities = ["read"]
}
# Get a consul token to access the kv store, where patroni will manage the leader lock
path "[[ .vault.prefix ]]consul/creds/[[ .pg.instance ]]" {
path "[[ .vault.prefix ]]consul/creds/[[ .instance ]]" {
capabilities = ["read"]
}
# Get a certificate for patroni REST API and Postgres
path "[[ .vault.prefix ]]pki/[[ .pg.instance ]]/issue/postgres-server" {
path "[[ .vault.prefix ]]pki/[[ .instance ]]/issue/postgres-server" {
capabilities = ["update"]
}