nomad
/
postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleanup

This commit is contained in:
Daniel Berteaud 2024-01-31 15:58:44 +01:00
parent 655e4a3389
commit 976480545d
20 changed files with 89 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
consul/policies/postgres.hcl
View File

@ -7,7 +7,7 @@ service "[[ .instance ]]" {
service "[[ .instance ]]-sidecar-proxy" {
policy = "write"
}
key_prefix "[[ .consul.prefix ]]service/[[ .instance ]]" {
key_prefix "[[ .consul.kv.root ]]service/[[ .instance ]]" {
policy = "write"
}
session_prefix "" {

2
example/init/consul
View File

@ -1,4 +1,4 @@
#!/bin/sh
# vim: syntax=sh
vault write consul/roles/postgres ttl=720h max_ttl=720h consul_policies="postgres"
vault write /consul/roles/postgres ttl=720h max_ttl=720h consul_policies="postgres"

11
example/init/passwords
View File

@ -1,11 +0,0 @@
#!/bin/sh
for USER in pg monitor replicator rewind api vault_initial; do
vault kv get -field ${USER}_pwd kv/service/postgres > /dev/null 2>&1
RES=$?
if [ "${RES}" = "1" ]; then
vault kv patch kv/service/postgres ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
elif [ "${RES}" = "2" ]; then
vault kv put kv/service/postgres ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
fi
done

40
example/init/pki
View File

@ -11,39 +11,39 @@ set -euo pipefail
TMP=$(mktemp -d)
INITIAL_SETUP=false
if [ "$(vault secrets list -format json | jq -r '.["pki/postgres/"].type')" != "pki" ]; then
if [ "$(vault secrets list -format json | jq -r '.["/pki/postgres/"].type')" != "pki" ]; then
INITIAL_SETUP=true
fi
if [ "${INITIAL_SETUP}" = "true" ]; then
# Enable the secret engine
echo "Mounting new PKI secret engine at pki/postgres"
vault secrets enable -path=pki/postgres pki
echo "Mounting new PKI secret engine at /pki/postgres"
vault secrets enable -path=/pki/postgres pki
else
echo "Secret engine already mounted at pki/postgres"
echo "Secret engine already mounted at /pki/postgres"
fi
# Configure max-lease-ttl
echo "Tune PKI secret engine"
vault secrets tune -max-lease-ttl=131400h pki/postgres
vault secrets tune -max-lease-ttl=131400h /pki/postgres
# Configure PKI URLs
echo "Configure URL endpoints"
vault write pki/postgres/config/urls \
issuing_certificates="${VAULT_ADDR}/v1/pki/postgres/ca" \
crl_distribution_points="${VAULT_ADDR}/v1/pki/postgres/crl" \
ocsp_servers="${VAULT_ADDR}/v1/pki/postgres/ocsp"
vault write /pki/postgres/config/urls \
issuing_certificates="${VAULT_ADDR}/v1//pki/postgres/ca" \
crl_distribution_points="${VAULT_ADDR}/v1//pki/postgres/crl" \
ocsp_servers="${VAULT_ADDR}/v1//pki/postgres/ocsp"
vault write pki/postgres/config/cluster \
path="${VAULT_ADDR}/v1/pki/postgres"
vault write /pki/postgres/config/cluster \
path="${VAULT_ADDR}/v1//pki/postgres"
vault write pki/postgres/config/crl \
vault write /pki/postgres/config/crl \
auto_rebuild=true \
enable_delta=true
# Configure tidy
echo "Configure auto tidy for the PKI"
vault write pki/postgres/config/auto-tidy \
vault write /pki/postgres/config/auto-tidy \
enabled=true \
tidy_cert_store=true \
tidy_expired_issuers=true \
@ -58,12 +58,12 @@ vault write pki/postgres/config/auto-tidy \
if [ "${INITIAL_SETUP}" = "true" ]; then
# Generate an internal CA
echo "Generating an internal CA"
vault write -format=json pki/postgres/intermediate/generate/internal \
vault write -format=json /pki/postgres/intermediate/generate/internal \
common_name="postgres Certificate Authority" \
ttl="131400h" \
organization="Ehtrace" \
organization="ACME Corp" \
ou="Postgres" \
locality="Pessac" \
locality="FooBar Ville" \
key_type=rsa \
key_bits=4096 \
| jq -r '.data.csr' > ${TMP}/postgres.csr
@ -71,8 +71,8 @@ if [ "${INITIAL_SETUP}" = "true" ]; then
# Sign this PKI with a root PKI
echo "Signing the new CA with the authority from pki/root"
vault write -format=json pki/root/root/sign-intermediate \
echo "Signing the new CA with the authority from /pki/root"
vault write -format=json /pki/root/root/sign-intermediate \
csr=@${TMP}/postgres.csr \
format=pem_bundle \
ttl="131400h" \
@ -80,7 +80,7 @@ if [ "${INITIAL_SETUP}" = "true" ]; then
# Update the intermediate CA with the signed one
echo "Update the new CA with the signed version"
vault write pki/postgres/intermediate/set-signed \
vault write /pki/postgres/intermediate/set-signed \
certificate=@${TMP}/postgres.crt
@ -91,7 +91,7 @@ echo "Cleaning temp files"
rm -rf ${TMP}
vault write pki/postgres/roles/postgres-server \
vault write /pki/postgres/roles/postgres-server \
allowed_domains="postgres.service.consul" \
allow_bare_domains=true \
allow_subdomains=true \

4
example/init/vault-database
View File

@ -1,9 +1,7 @@
#!/bin/sh
echo "Required .pg.server.public_url is missing"
echo "Creating dba role in vault"
vault write database/roles/postgres-admin \
echo "Creating dba role in vault"vault write /database/roles/postgres-admin \
db_name="postgres" \
creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \
GRANT \"dba\" TO \"{{name}}\"; \

2
example/manage.nomad.hcl
View File

@ -121,7 +121,7 @@ _EOT
PGHOST=localhost
PGPORT=5432
PGUSER=postgres
PGPASSWORD={{ with secret "kv/service/postgres" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
PGPASSWORD={{ with secret "/kv/service/postgres" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
_EOF
destination = "secrets/pg-manage.env"
uid = 100000

12
example/postgres.nomad.hcl
View File

@ -168,7 +168,7 @@ _EOT
template {
data = <<_EOT
# Get a Consul token from vault, so we're able to update the tags in Consul from the containers
CONSUL_HTTP_TOKEN={{ with secret "consul/creds/postgres" }}{{ .Data.token }}{{ end }}
CONSUL_HTTP_TOKEN={{ with secret "/consul/creds/postgres" }}{{ .Data.token }}{{ end }}
PATRONICTL_CONFIG_FILE=/secrets/patroni.yml
_EOT
destination = "secrets/pg.env"
@ -347,7 +347,7 @@ postgresql:
authentication:
superuser:
username: postgres
password: '{{ with secret "kv/service/postgres" }}{{ .Data.data.pg_pwd }}{{ end }}'
password: '{{ with secret "/kv/service/postgres" }}{{ .Data.data.pg_pwd }}{{ end }}'
sslmode: verify-ca
sslrootcert: /local/postgres.ca.pem
@ -374,7 +374,7 @@ restapi:
verify_client: optional
authentication:
username: patroni
password: '{{ with secret "kv/service/postgres" }}{{ .Data.data.api_pwd }}{{ end }}'
password: '{{ with secret "/kv/service/postgres" }}{{ .Data.data.api_pwd }}{{ end }}'
ctl:
insecure: False
@ -402,7 +402,7 @@ _EOT
set -euo pipefail
# Create roles needed for patroni
{{ with secret "kv/service/postgres" }}
{{ with secret "/kv/service/postgres" }}
psql <<'_EOSQL'
ALTER ROLE postgres WITH SUPERUSER LOGIN PASSWORD '{{ .Data.data.pg_pwd }}';
CREATE ROLE replicator WITH LOGIN REPLICATION PASSWORD '{{ .Data.data.replicator_pwd }}';
@ -439,7 +439,7 @@ _EOT
template {
data = <<_EOT
{{ with pkiCert
"pki/postgres/issue/postgres-server"
"/pki/postgres/issue/postgres-server"
"common_name=postgres.service.consul"
(printf "alt_name=%s.postgres.service.consul" (env "NOMAD_ALLOC_INDEX"))
(printf "ip_sans=%s" (env "NOMAD_IP_patroni")) "ttl=72h" }}
@ -458,7 +458,7 @@ _EOT
# CA certificate chains
template {
data = <<_EOT
{{ with secret "pki/postgres/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
{{ with secret "/pki/postgres/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
destination = "local/postgres.ca.pem"
change_mode = "signal"

33
example/prep.d/10-rand-pwd.sh
View File

@ -2,21 +2,26 @@
set -euo pipefail
# Initialize random passwords if needed
# vim: syntax=sh
export LC_ALL=C
VAULT_KV_PATH=/kv/service/postgres
RAND_CMD="tr -dc A-Za-z0-9\-_\/=~\.+ < /dev/urandom | head -c 50"
if ! vault kv list $(dirname ${VAULT_KV_PATH}) 2>/dev/null | grep -q -E "^$(basename ${VAULT_KV_PATH})\$"; then
vault kv put ${VAULT_KV_PATH} \
pg_pwd="$(sh -c "${RAND_CMD}")" \
api_pwd="$(sh -c "${RAND_CMD}")" \
monitor_pwd="$(sh -c "${RAND_CMD}")" \
replicator_pwd="$(sh -c "${RAND_CMD}")" \
rewind_pwd="$(sh -c "${RAND_CMD}")" \
vault_initial_pwd="$(sh -c "${RAND_CMD}")" \
if ! vault kv list kv/service 2>/dev/null | grep -q -E '^postgres$'; then
vault kv put kv/service/postgres \
pg_pwd=$(pwgen -s -n 50 1) \
api_pwd=$(pwgen -s -n 50 1) \
monitor_pwd=$(pwgen -s -n 50 1) \
replicator_pwd=$(pwgen -s -n 50 1) \
rewind_pwd=$(pwgen -s -n 50 1) \
vault_initial_pwd=$(pwgen -s -n 50 1)
fi
for PWD in pg_pwd api_pwd monitor_pwd replicator_pwd rewind_pwd vault_initial_pwd; do
if ! vault kv get -field ${PWD} kv/service/postgres >/dev/null 2>&1; then
vault kv patch kv/service/postgres \
${PWD}=$(pwgen -s -n 50 1)
for SECRET in pg_pwd api_pwd monitor_pwd replicator_pwd rewind_pwd vault_initial_pwd; do
if ! vault kv get -field ${SECRET} ${VAULT_KV_PATH} >/dev/null 2>&1; then
vault kv patch ${VAULT_KV_PATH} \
${SECRET}=$(sh -c "${RAND_CMD}")
fi
done

19
example/prep.d/mv_conf.sh
View File

@ -1,19 +0,0 @@
#!/bin/sh
set -eu
if [ "postgres" != "postgres" ]; then
for DIR in vault consul nomad; do
if [ -d output/${DIR} ]; then
for FILE in $(find output/${DIR} -name "*postgres*.hcl" -type f); do
NEW_FILE=$(echo "${FILE}" | sed -E "s/postgres/postgres/g")
mv "${FILE}" "${NEW_FILE}"
done
fi
done
fi

6
example/vault/policies/postgres.hcl
View File

@ -1,14 +1,14 @@
# Read secrets from vault KV
path "kv/data/service/postgres" {
path "/kv/data/service/postgres" {
capabilities = ["read"]
}
# Get a consul token to access the kv store, where patroni will manage the leader lock
path "consul/creds/postgres" {
path "/consul/creds/postgres" {
capabilities = ["read"]
}
# Get a certificate for patroni REST API and Postgres
path "pki/postgres/issue/postgres-server" {
path "/pki/postgres/issue/postgres-server" {
capabilities = ["update"]
}

2
init/consul
View File

@ -1,4 +1,4 @@
#!/bin/sh
# vim: syntax=sh
vault write [[ .vault.prefix ]]consul/roles/[[ .instance ]] ttl=720h max_ttl=720h consul_policies="[[ .instance ]]"
vault write [[ .vault.root ]]consul/roles/[[ .instance ]] ttl=720h max_ttl=720h consul_policies="[[ .instance ]]"

11
init/passwords
View File

@ -1,11 +0,0 @@
#!/bin/sh
for USER in pg monitor replicator rewind api vault_initial; do
vault kv get -field ${USER}_pwd [[ .vault.kv.path ]] > /dev/null 2>&1
RES=$?
if [ "${RES}" = "1" ]; then
vault kv patch [[ .vault.kv.path ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
elif [ "${RES}" = "2" ]; then
vault kv put [[ .vault.kv.path ]] ${USER}_pwd=$(pwgen -s -y -r\\\`\'\"\#\^\| -n 50 1)
fi
done

25
init/vault-database
View File

@ -1,34 +1,31 @@
#!/bin/sh
[[- if has .pg.server "public_url" ]]
if [ "$(vault secrets list -format json | jq -r '.["[[ .vault.prefix ]]database/"].type')" != "database" ]; then
echo "Enabling database secret on [[ .vault.prefix ]]database"
vault secrets enable -path [[ .vault.prefix ]]database database
if [ "$(vault secrets list -format json | jq -r '.["[[ .vault.root ]]database/"].type')" != "database" ]; then
echo "Enabling database secret on [[ .vault.root ]]database"
vault secrets enable -path [[ .vault.root ]]database database
else
echo "Database secret already enabled at [[ .vault.prefix ]]database"
echo "Database secret already enabled at [[ .vault.root ]]database"
fi
if [ "$(vault list -format json [[ .vault.prefix ]]database/config | jq '.[] | test("^[[ .instance ]]$")')" != "true" ]; then
echo "Configuring database plugin [[ .vault.prefix ]]database/config/[[ .instance ]]"
vault write [[ .vault.prefix ]]database/config/[[ .instance ]] \
if [ "$(vault list -format json [[ .vault.root ]]database/config | jq '.[] | test("^[[ .instance ]]$")')" != "true" ]; then
echo "Configuring database plugin [[ .vault.root ]]database/config/[[ .instance ]]"
vault write [[ .vault.root ]]database/config/[[ .instance ]] \
plugin_name="postgresql-database-plugin" \
connection_url="postgresql://{{username}}:{{password}}@[[ (urlParse .pg.server.public_url).Host ]]/postgres" \
allowed_roles="*" \
username=vault \
password="$(vault kv get -field vault_initial_pwd [[ .vault.prefix ]]kv/service/[[ .instance ]])" \
password="$(vault kv get -field vault_initial_pwd [[ .vault.root ]]kv/service/[[ .instance ]])" \
password_authentication=scram-sha-256 \
disable_escaping=true
echo "Rotating root password"
vault write -force [[ .vault.prefix ]]database/rotate-root/[[ .instance ]]
vault write -force [[ .vault.root ]]database/rotate-root/[[ .instance ]]
else
echo "Database plugin already configured for [[ .vault.prefix ]]database/config/[[ .instance ]]"
echo "Database plugin already configured for [[ .vault.root ]]database/config/[[ .instance ]]"
fi
[[- else ]]
echo "Required .pg.server.public_url is missing"
[[- end ]]
echo "Creating dba role in vault"
[[- template "common/vault.mkpgrole.sh.tpl"
dict "ctx" .
"config" (dict "role" (printf "%s-admin" .instance) "pgrole" "dba" "database" .instance)
]]
[[- template "common/vault.mkpgrole.sh.tpl" merge .pg . ]]

2
manage.nomad.hcl
View File

@ -75,7 +75,7 @@ job "[[ .instance ]]-manage" {
PGHOST=localhost
PGPORT=5432
PGUSER=postgres
PGPASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
PGPASSWORD={{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd | sprig_squote }}{{ end }}
_EOF
destination = "secrets/pg-manage.env"
uid = 100000

4
postgres.nomad.hcl
View File

@ -146,7 +146,7 @@ job "[[ .instance ]]" {
template {
data = <<_EOT
# Get a Consul token from vault, so we're able to update the tags in Consul from the containers
CONSUL_HTTP_TOKEN={{ with secret "[[ .vault.prefix ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
CONSUL_HTTP_TOKEN={{ with secret "[[ .vault.root ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
PATRONICTL_CONFIG_FILE=/secrets/patroni.yml
_EOT
destination = "secrets/pg.env"
@ -360,7 +360,7 @@ _EOT
PG_EXPORTER_AUTO_DISCOVER_DATABASES=true
DATA_SOURCE_URI=127.0.0.1:5432/postgres
DATA_SOURCE_USER=monitor
DATA_SOURCE_PASS={{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.monitor_pwd | sprig_squote }}{{ end }}
DATA_SOURCE_PASS={{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.monitor_pwd | sprig_squote }}{{ end }}
_EOT
destination = "secrets/env"
perms = "0400"

19
prep.d/10-rand-pwd.sh
View File

@ -2,21 +2,4 @@
set -euo pipefail
# Initialize random passwords if needed
if ! vault kv list [[ .vault.prefix ]]kv/service 2>/dev/null | grep -q -E '^[[ .instance ]]$'; then
vault kv put [[ .vault.prefix ]]kv/service/[[ .instance ]] \
pg_pwd=$(pwgen -s -n 50 1) \
api_pwd=$(pwgen -s -n 50 1) \
monitor_pwd=$(pwgen -s -n 50 1) \
replicator_pwd=$(pwgen -s -n 50 1) \
rewind_pwd=$(pwgen -s -n 50 1) \
vault_initial_pwd=$(pwgen -s -n 50 1)
fi
for PWD in pg_pwd api_pwd monitor_pwd replicator_pwd rewind_pwd vault_initial_pwd; do
if ! vault kv get -field ${PWD} [[ .vault.prefix ]]kv/service/[[ .instance ]] >/dev/null 2>&1; then
vault kv patch [[ .vault.prefix ]]kv/service/[[ .instance ]] \
${PWD}=$(pwgen -s -n 50 1)
fi
done
[[ template "common/vault.rand_secrets" merge .pg . ]]

2
templates/create_users.sh.tpl
View File

@ -3,7 +3,7 @@
set -euo pipefail
# Create roles needed for patroni
{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}
{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}
psql <<'_EOSQL'
ALTER ROLE postgres WITH SUPERUSER LOGIN PASSWORD '{{ .Data.data.pg_pwd }}';
CREATE ROLE replicator WITH LOGIN REPLICATION PASSWORD '{{ .Data.data.replicator_pwd }}';

6
templates/patroni.yml.tpl
View File

@ -3,7 +3,7 @@ scope: [[ .instance ]]
consul:
url: http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500
token: {{ with secret "[[ .consul.prefix ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
token: {{ with secret "[[ .consul.kv.root ]]consul/creds/[[ .instance ]]" }}{{ .Data.token }}{{ end }}
bootstrap:
dcs:
@ -109,7 +109,7 @@ postgresql:
authentication:
superuser:
username: postgres
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd }}{{ end }}'
password: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.pg_pwd }}{{ end }}'
sslmode: verify-ca
sslrootcert: /local/postgres.ca.pem
@ -136,7 +136,7 @@ restapi:
verify_client: optional
authentication:
username: patroni
password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.api_pwd }}{{ end }}'
password: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.api_pwd }}{{ end }}'
ctl:
insecure: False

18
variables.yml
View File

@ -8,14 +8,26 @@ pg:
vault:
pki:
path: '[[ .vault.prefix ]]pki/[[ .instance ]]'
ou: Postgres
issuer: '[[ .vault.prefix ]]pki/root'
# List of vault policies to attach to the task
policies:
- '[[ .instance ]][[ .consul.suffix ]]'
database:
role: '[[ .instance ]]-admin'
pgrole: dba
# Random secrets to generate if missing, and store in vault KV
rand_secrets:
fields:
- pg_pwd
- api_pwd
- monitor_pwd
- replicator_pwd
- rewind_pwd
- vault_initial_pwd
# Postgres server settings
server:
# The image to use
@ -110,7 +122,7 @@ pg:
# Optional bind DN and password to do the search operation
# If undefined, the search will be done anonymously
#bind_dn: CN=Postgres,OU=Apps,DC=example,DC=org
bind_password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.ldap_pwd }}{{ end }}'
bind_password: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.ldap_pwd }}{{ end }}'
# Resource allocation for the container
resources:

4
vault/policies/postgres.hcl
View File

@ -1,10 +1,10 @@
# Read secrets from vault KV
path "[[ .vault.prefix ]]kv/data/service/[[ .instance ]]" {
path "[[ .vault.root ]]kv/data/service/[[ .instance ]]" {
capabilities = ["read"]
}
# Get a consul token to access the kv store, where patroni will manage the leader lock
path "[[ .vault.prefix ]]consul/creds/[[ .instance ]]" {
path "[[ .vault.root ]]consul/creds/[[ .instance ]]" {
capabilities = ["read"]
}