Progress on pgman, to manage db and privileges

consul/config/service-intentions/postgres.hcl

@@ -0,0 +1,8 @@
+Kind = "service-intentions"
+Name = "[[ .pg.job_name ]][[ .env.suffix ]]"
+Sources = [
+  {
+    Name = "[[ .pg.job_name ]]-manager[[ .env.suffix ]]"
+    Action = "allow"
+  }
+]

deps/common

@@ -1 +1 @@
-Subproject commit edb5b094c53c28aee30ffda395042f88960febfb
+Subproject commit 54d69c4b349be1688c31a161cf0591bfc1bf29fd

images/pgman/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine AS ldap2pg
+
+ARG LDAP2PG_VERSION=6.0
+
+RUN set -eux &&\
+    cd /tmp &&\
+    apk --no-cache add ca-certificates curl &&\
+    curl -sSLO https://github.com/dalibo/ldap2pg/releases/download/v${LDAP2PG_VERSION}/ldap2pg_${LDAP2PG_VERSION}_linux_amd64.tar.gz &&\
+    curl -sSLO  https://github.com/dalibo/ldap2pg/releases/download/v${LDAP2PG_VERSION}/ldap2pg_${LDAP2PG_VERSION}_checksums.txt &&\
+    grep ldap2pg_${LDAP2PG_VERSION}_linux_amd64.tar.gz ldap2pg_${LDAP2PG_VERSION}_checksums.txt | sha256sum -c &&\
+    tar xvzf ldap2pg_${LDAP2PG_VERSION}_linux_amd64.tar.gz &&\
+    chown root:root ldap2pg &&\
+    chmod 755 ldap2pg
+
+FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]]
+MAINTAINER [[ .docker.maintainer ]]
+
+COPY --from=ldap2pg /tmp/ldap2pg /usr/local/bin/ldap2pg
+RUN set -eux &&\
+    apk --no-cache upgrade &&\
+    apk --no-cache add postgresql15-client ca-certificates
+COPY root/ /
+
+CMD ["pgman"]

images/pgman/root/usr/local/bin/pgman

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -x
+
+for IDX in $(printenv | grep -E '^PG_DB_([0-9]+)=' | sed -E 's/^PG_DB_([0-9]+)=.*/\1/'); do
+  DB_NAME=$(printenv PG_DB_${IDX})
+  echo "Found DB ${DB_NAME} to create"
+  DB_OWNER=$(printenv PG_DB_${IDX}_OWNER || echo "")
+  DB_ENCODING=$(printenv PG_DB_${IDX}_ENCODING || echo "")
+  DB_LOCALE=$(printenv PG_DB_${IDX}_LOCALE || echo "")
+  CMD="createdb"
+  if [ -n "${DB_OWNER}" ]; then
+    CMD="${CMD} --owner=${DB_OWNER}"
+  fi
+  if [ -n "${DB_ENCODING}" ]; then
+    CMD="${CMD} --encoding=${DB_ENCODING}"
+  fi
+  if [ -n "${DB_LOCALE}" ]; then
+    CMD="${CMD} --locale=${DB_LOCALE}"
+  fi
+  CMD="${CMD} ${DB_NAME}"
+  echo "Creating DB ${DB_NAME}"
+  ${CMD} || echo $?
+done
+
+if [ -e "${LDAP2PG_CONFIG}" ]; then
+  if [ "${LDAP2PG_MODE}" = "dry" ]; then
+    echo "Running ldap2pg in dry mode"
+    ldap2pg --config ${LDAP2PG_CONFIG}
+  elif [ "${LDAP2PG_MODE}" = "real" ]; then
+    echo "Applying privileges with ldap2pg"
+    ldap2pg --real --config ${LDAP2PG_CONFIG}
+  fi
+fi

pgman.nomad.hcl

@@ -0,0 +1,89 @@
+job "[[ .pg.job_name ]]-manager[[ .env.suffix ]]" {
+
+  type = "batch"
+
+[[ template "common/job_start.tpl" . ]]
+
+  meta {
+    # Force job to run each time
+    run = "${uuidv4()}"
+  }
+
+  group "pgman" {
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "[[ .pg.job_name ]]-manager[[ .env.suffix ]]"
+[[ template "common/connect.tpl" dict "ctx" . "config" .pg.pgman ]]
+    }
+
+    task "pgman" {
+      driver = [[ .pg.pgman.driver | toJSON ]]
+
+      config {
+        image           = [[ .pg.pgman.image | toJSON ]]
+        readonly_rootfs = true
+        pids_limit      = 20
+      }
+
+      vault {
+        policies     = ["[[ .pg.job_name ]][[ .env.suffix ]]"]
+        env          = false
+        disable_file = true
+      }
+
+      env {
+[[- range $idx, $db := .pg.pgman.databases ]]
+        PG_DB_[[ $idx ]] = [[ $db.name | toJSON ]]
+  [[- if has $db "owner" ]]
+        PG_DB_[[ $idx ]]_OWNER = [[ $db.owner | toJSON ]]
+  [[- end ]]
+  [[- if has $db "encoding" ]]
+        PG_DB_[[ $idx ]]_ENCODING = [[ $db.encoding | toJSON ]]
+  [[- end ]]
+  [[- if has $db "locale" ]]
+        PG_DB_[[ $idx ]]_OWNER = [[ $db.locale | toJSON ]]
+  [[- end ]]
+[[- end ]]
+        LDAP2PG_CONFIG = "/secrets/ldap2pg.yml"
+        LDAP2PG_MODE = var.ldap2pg
+      }
+
+      template {
+        data =<<_EOF
+PGHOST=localhost
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.job_name ]]" }}{{ .Data.data.pg_pwd }}{{ end }}
+_EOF
+        destination = "secrets/env"
+        uid = 100000
+        gid = 100000
+        perms = 0400
+        env = true
+      }
+
+      template {
+        data =<<_EOF
+[[ (merge .pg.pgman.ldap2pg.config ((tmpl.Exec "postgres/ldap2pg.yml.tpl" .) | yaml)) | toYAML ]]
+
+rules:
+[[ (coll.Slice ((tmpl.Exec "postgres/ldap2pg_rules.yml.tpl" .) | yaml).rules (.pg.pgman.ldap2pg.rules)) | flatten | toYAML | strings.Indent 2]]
+_EOF
+        destination = "secrets/ldap2pg.yml"
+        uid = 100000
+        gid = 100000
+        perms = 0400
+      }
+
+[[ template "common/resources.tpl" .pg.pgman.resources ]]
+    }
+  }
+}
+
+variable "ldap2pg" {
+  type    = string
+  default = "dry"
+}

postgres.nomad.hcl

@@ -33,6 +33,7 @@ job [[ .pg.job_name | toJSON ]] {
 
       config {
         image = [[ .pg.server.image | toJSON ]]
+        pids_limit = 700
       }
 
       vault {
@@ -85,6 +86,8 @@ job [[ .pg.job_name | toJSON ]] {
         args = [
           "--web.listen-address=127.0.0.1:9187"
         ]
+        readonly_rootfs = true
+        pids_limit      = 20
       }
 
       vault {
@@ -113,4 +116,27 @@ job [[ .pg.job_name | toJSON ]] {
     }
 [[- end ]]
   }
+
+  group "pgman" {
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "[[ .pg.job_name ]]-manager[[ .env.suffix ]]"
+[[ template "common/connect.tpl" dict "ctx" . "config" .pg.pgman ]]
+    }
+
+    task "pgman" {
+      driver = [[ .pg.pgman.driver | toJSON ]]
+
+      config {
+        image = [[ .pg.pgman.image | toJSON ]]
+        readonly_rootfs = true
+        pids_limit      = 20
+      }
+
+[[ template "common/resources.tpl" .pg.pgman.resources ]]
+    }
+  }
 }

templates/group.ldap2pg.tpl

@@ -0,0 +1,75 @@
+# vim: syntax=hcl
+
+group "pgman" {
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "[[ .pg.job_name ]]-manager[[ .env.suffix ]]"
+[[ template "common/connect.tpl" dict "ctx" . "config" .pg.pgman ]]
+    }
+
+    task "pgman" {
+      driver = [[ .pg.pgman.driver | toJSON ]]
+
+      config {
+        image           = [[ .pg.pgman.image | toJSON ]]
+        readonly_rootfs = true
+        pids_limit      = 20
+      }
+
+      vault {
+        policies     = ["[[ .pg.job_name ]][[ .env.suffix ]]"]
+        env          = false
+        disable_file = true
+      }
+
+      env {
+[[- range $idx, $db := .pg.pgman.databases ]]
+        PG_DB_[[ $idx ]] = [[ $db.name | toJSON ]]
+  [[- if has $db "owner" ]]
+        PG_DB_[[ $idx ]]_OWNER = [[ $db.owner | toJSON ]]
+  [[- end ]]
+  [[- if has $db "encoding" ]]
+        PG_DB_[[ $idx ]]_ENCODING = [[ $db.encoding | toJSON ]]
+  [[- end ]]
+  [[- if has $db "locale" ]]
+        PG_DB_[[ $idx ]]_OWNER = [[ $db.locale | toJSON ]]
+  [[- end ]]
+[[- end ]]
+        LDAP2PG_CONFIG = "/secrets/ldap2pg.yml"
+        LDAP2PG_MODE = var.ldap2pg
+      }
+
+      template {
+        data =<<_EOF
+PGHOST=localhost
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .pg.job_name ]]" }}{{ .Data.data.pg_pwd }}{{ end }}
+_EOF
+        destination = "secrets/env"
+        uid = 100000
+        gid = 100000
+        perms = 0400
+        env = true
+      }
+
+      template {
+        data =<<_EOF
+[[ (merge .pg.pgman.ldap2pg.config ((tmpl.Exec "postgres/ldap2pg.yml.tpl" .) | yaml)) | toYAML ]]
+
+rules:
+[[ (coll.Slice ((tmpl.Exec "postgres/ldap2pg_rules.yml.tpl" .) | yaml).rules (.pg.pgman.ldap2pg.rules)) | flatten | toYAML | strings.Indent 2]]
+_EOF
+        destination = "secrets/ldap2pg.yml"
+        uid = 100000
+        gid = 100000
+        perms = 0400
+      }
+
+[[ template "common/resources.tpl" .pg.pgman.resources ]]
+    }
+  }
+

templates/ldap2pg.yml.tpl

@@ -0,0 +1,44 @@
+---
+
+# vim: syntax=yaml
+
+version: 6
+
+postgres:
+  managed_roles_query: |
+    VALUES
+      ('public'),
+      ('managed_roles')
+
+    UNION
+
+    SELECT DISTINCT role.rolname
+    FROM pg_roles AS role
+    JOIN pg_auth_members AS ms ON ms.member = role.oid
+    JOIN pg_roles AS parent
+      ON parent.rolname = 'managed_roles' AND parent.oid = ms.roleid
+    ORDER BY 1;
+
+privileges:
+  reader:
+    - __connect__
+    - __usage_on_schemas__
+    - __select_on_tables__
+    - __select_on_sequences__
+    - __usage_on_sequences__
+
+  writer:
+    - reader
+    - __temporary__
+    - __insert_on_tables__
+    - __update_on_tables__
+    - __delete__on_tables__
+    - __update_on_sequences__
+    - __execute_on_functions__
+    - __trigger_on_tables__
+
+  owner:
+    - writer
+    - __create_on_schemas__
+    - __truncate_on_tables__
+

templates/ldap2pg_rules.yml.tpl

@@ -0,0 +1,37 @@
+---
+# vim: syntax=yaml
+
+rules:
+  - roles:
+      - name: managed_roles
+        comment: Parent role for all ldap2pg managed roles
+  
+      - name: ldap_roles
+        comment: "Parent role for LDAP synced roles"
+        options: NOLOGIN
+        parents:
+          - managed_roles
+  
+      - name: backup
+        comment: "DB backup"
+        parents:
+          - pg_read_all_data
+          - managed_roles
+  
+      - name: monitor
+        comment: "Monitoring"
+        parents:
+          - pg_monitor
+          - managed_roles
+  
+      - name: replicator
+        comment: Postgres replication account
+        options: LOGIN REPLICATION
+        parents:
+          - managed_roles
+  
+      - name: dba
+        comment: "Databases admins"
+        options: SUPERUSER NOLOGIN
+        parents: managed_roles
+

variables.yml

@@ -30,3 +30,19 @@ pg:
       cpu: 100
       memory: 256
     env: {}
+
+  pgman:
+    image: danielberteaud/pgman:latest
+    driver: docker
+    resources:
+      cpu: 20
+      memory: 32
+    env: {}
+    connect:
+      upstreams:
+        - destination_name: postgres
+          local_bind_port: 5432
+    databases: []
+    ldap2pg:
+      config: {}
+      rules: []