Support postgres pooler

2024-01-12 21:52:20 +01:00 · 2024-01-12 21:52:20 +01:00 · 6a8c74b6c4
parent 49915ad8fa
commit 6a8c74b6c4
4 changed files with 118 additions and 48 deletions
--- a/example/paperless-ngx.nomad.hcl
+++ b/example/paperless-ngx.nomad.hcl
@ -118,6 +118,8 @@ _EOT



+
+
    task "webserver" {
      driver = "docker"

@ -127,12 +129,14 @@ _EOT
        pids_limit      = 200
      }

+
      vault {
        policies     = ["paperless"]
        env          = false
        disable_file = true
      }

+
      env {
        PAPERLESS_MODE      = "webserver"
        PAPERLESS_BIND_ADDR = "127.0.0.1"
@ -146,9 +150,6 @@ _EOT
 LANG=fr_FR.utf8
 PAPERLESS_CONVERT_TMPDIR=/alloc/data
 PAPERLESS_CORS_ALLOWED_HOSTS=https://paperless.example.org
-PAPERLESS_DBNAME=paperless
-PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
-PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
 PAPERLESS_ENABLE_COMPRESSION=false
 PAPERLESS_PROXY_SSL_HEADER='["HTTP_X_FORWARDED_PROTO", "https"]'
 PAPERLESS_SECRET_KEY={{ with secret "kv/service/paperless" }}{{ .Data.data.secret_key }}{{ end }}
@ -163,6 +164,19 @@ _EOT
      }


+      template {
+        data        = <<_EOT
+PAPERLESS_DBHOST=127.0.0.1
+PAPERLESS_DBPORT=5432
+PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
+PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
+_EOT
+        destination = "secrets/.db.env"
+        perms       = 400
+        env         = true
+      }
+
+
      volume_mount {
        volume      = "data"
        destination = "/data"
@ -184,7 +198,7 @@ _EOT
      driver = "docker"

      lifecycle {
-        hook    = "prestart"
+        hook    = "poststart"
        sidecar = true
      }

@ -194,12 +208,14 @@ _EOT
        pids_limit      = 100
      }

+
      vault {
        policies     = ["paperless"]
        env          = false
        disable_file = true
      }

+
      env {
        PAPERLESS_MODE = "consumer"
        TMPDIR         = "/alloc/tmp"
@ -212,9 +228,6 @@ _EOT
 LANG=fr_FR.utf8
 PAPERLESS_CONVERT_TMPDIR=/alloc/data
 PAPERLESS_CORS_ALLOWED_HOSTS=https://paperless.example.org
-PAPERLESS_DBNAME=paperless
-PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
-PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
 PAPERLESS_ENABLE_COMPRESSION=false
 PAPERLESS_PROXY_SSL_HEADER='["HTTP_X_FORWARDED_PROTO", "https"]'
 PAPERLESS_SECRET_KEY={{ with secret "kv/service/paperless" }}{{ .Data.data.secret_key }}{{ end }}
@ -229,6 +242,19 @@ _EOT
      }


+      template {
+        data        = <<_EOT
+PAPERLESS_DBHOST=127.0.0.1
+PAPERLESS_DBPORT=5432
+PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
+PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
+_EOT
+        destination = "secrets/.db.env"
+        perms       = 400
+        env         = true
+      }
+
+
      volume_mount {
        volume      = "data"
        destination = "/data"
@ -261,12 +287,14 @@ _EOT
        pids_limit      = 100
      }

+
      vault {
        policies     = ["paperless"]
        env          = false
        disable_file = true
      }

+
      env {
        PAPERLESS_MODE = "scheduler"
        TMPDIR         = "/alloc/tmp"
@ -279,9 +307,6 @@ _EOT
 LANG=fr_FR.utf8
 PAPERLESS_CONVERT_TMPDIR=/alloc/data
 PAPERLESS_CORS_ALLOWED_HOSTS=https://paperless.example.org
-PAPERLESS_DBNAME=paperless
-PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
-PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
 PAPERLESS_ENABLE_COMPRESSION=false
 PAPERLESS_PROXY_SSL_HEADER='["HTTP_X_FORWARDED_PROTO", "https"]'
 PAPERLESS_SECRET_KEY={{ with secret "kv/service/paperless" }}{{ .Data.data.secret_key }}{{ end }}
@ -296,6 +321,19 @@ _EOT
      }


+      template {
+        data        = <<_EOT
+PAPERLESS_DBHOST=127.0.0.1
+PAPERLESS_DBPORT=5432
+PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
+PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
+_EOT
+        destination = "secrets/.db.env"
+        perms       = 400
+        env         = true
+      }
+
+
      volume_mount {
        volume      = "data"
        destination = "/data"
@ -318,7 +356,7 @@ _EOT
      driver = "docker"

      lifecycle {
-        hook    = "prestart"
+        hook    = "poststart"
        sidecar = true
      }

@ -328,12 +366,14 @@ _EOT
        pids_limit      = 300
      }

+
      vault {
        policies     = ["paperless"]
        env          = false
        disable_file = true
      }

+
      env {
        PAPERLESS_MODE = "task-queue"
        TMPDIR         = "/alloc/tmp"
@ -346,9 +386,6 @@ _EOT
 LANG=fr_FR.utf8
 PAPERLESS_CONVERT_TMPDIR=/alloc/data
 PAPERLESS_CORS_ALLOWED_HOSTS=https://paperless.example.org
-PAPERLESS_DBNAME=paperless
-PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
-PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
 PAPERLESS_ENABLE_COMPRESSION=false
 PAPERLESS_PROXY_SSL_HEADER='["HTTP_X_FORWARDED_PROTO", "https"]'
 PAPERLESS_SECRET_KEY={{ with secret "kv/service/paperless" }}{{ .Data.data.secret_key }}{{ end }}
@ -363,6 +400,19 @@ _EOT
      }


+      template {
+        data        = <<_EOT
+PAPERLESS_DBHOST=127.0.0.1
+PAPERLESS_DBPORT=5432
+PAPERLESS_DBUSER={{ with secret "database/creds/paperless" }}{{ .Data.username }}{{ end }}
+PAPERLESS_DBPASS={{ with secret "database/creds/paperless" }}{{ .Data.password }}{{ end }}
+_EOT
+        destination = "secrets/.db.env"
+        perms       = 400
+        env         = true
+      }
+
+
      volume_mount {
        volume      = "data"
        destination = "/data"
--- a/paperless-ngx.nomad.hcl
+++ b/paperless-ngx.nomad.hcl
@ -35,7 +35,7 @@ job "[[ .instance ]]" {

 [[ template "common/connect" $c ]]

-[[ $c = merge .paperless.webserver . ]]
+[[ $c = merge .paperless.webserver .paperless . ]]
      tags = [
        "[[ $c.traefik.instance ]].enable=[[ if $c.traefik.enabled ]]true[[ else ]]false[[ end ]]",
        "[[ $c.traefik.instance ]].http.routers.[[ .instance ]][[ .consul.suffix ]].rule=Host(`[[ (urlParse $c.public_url).Hostname ]]`)
@ -50,8 +50,9 @@ job "[[ .instance ]]" {
      ]
    }

-[[ template "common/task.wait_for" merge .paperless . ]]
+[[ template "common/task.wait_for" $c ]]
 [[ template "common/task.redis" ]]
+[[ template "common/postgres_pooler" $c ]]

    task "webserver" {
      driver = [[ $c.nomad.driver | toJSON ]]
@ -62,11 +63,7 @@ job "[[ .instance ]]" {
        pids_limit      = 200
      }

-      vault {
-        policies     = ["[[ .instance ]][[ .consul.suffix ]]"]
-        env          = false
-        disable_file = true
-      }
+[[ template "common/vault.policies" $c ]]

      env {
        PAPERLESS_MODE      = "webserver"
@ -74,7 +71,8 @@ job "[[ .instance ]]" {
        TMPDIR              = "/alloc/tmp"
      }

-[[ template "common/file_env" merge $c.env .paperless.env ]]
+[[ template "common/file_env" $c.env ]]
+[[ template "paperless-ngx/dbenv" $c ]]

      volume_mount {
        volume      = "data"
@ -90,12 +88,12 @@ job "[[ .instance ]]" {
    }

 [[- if .paperless.consumer.enabled ]]
-  [[ $c := merge .paperless.consumer . ]]
+  [[ $c := merge .paperless.consumer .paperless . ]]
    task "consumer" {
      driver = [[ $c.nomad.driver | toJSON ]]

      lifecycle {
-        hook    = "prestart"
+        hook    = "poststart"
        sidecar = true
      }

@ -105,18 +103,15 @@ job "[[ .instance ]]" {
        pids_limit      = 100
      }

-      vault {
-        policies     = ["[[ .instance ]][[ .consul.suffix ]]"]
-        env          = false
-        disable_file = true
-      }
+[[ template "common/vault.policies" $c ]]

      env {
        PAPERLESS_MODE = "consumer"
        TMPDIR         = "/alloc/tmp"
      }

-  [[ template "common/file_env" merge $c.env .paperless.env ]]
+  [[ template "common/file_env" $c.env ]]
+  [[ template "paperless-ngx/dbenv" $c ]]

      volume_mount {
        volume      = "data"
@ -132,7 +127,7 @@ job "[[ .instance ]]" {
    }
 [[- end ]]

-[[ $c := merge .paperless.scheduler . ]]
+[[ $c := merge .paperless.scheduler .paperless . ]]
    task "scheduler" {
      driver = [[ $c.nomad.driver | toJSON ]]

@ -147,18 +142,15 @@ job "[[ .instance ]]" {
        pids_limit      = 100
      }

-      vault {
-        policies     = ["[[ .instance ]][[ .consul.suffix ]]"]
-        env          = false
-        disable_file = true
-      }
+[[ template "common/vault.policies" $c ]]

      env {
        PAPERLESS_MODE = "scheduler"
        TMPDIR         = "/alloc/tmp"
      }

-[[ template "common/file_env" merge $c.env .paperless.env ]]
+[[ template "common/file_env" $c.env ]]
+[[ template "paperless-ngx/dbenv" $c ]]

      volume_mount {
        volume      = "data"
@ -173,12 +165,12 @@ job "[[ .instance ]]" {
 [[ template "common/resources" $c.resources ]]
    }

-[[ $c := merge .paperless.task_queue . ]]
+[[ $c := merge .paperless.task_queue .paperless . ]]
    task "task-queue" {
      driver = [[ $c.nomad.driver | toJSON ]]

      lifecycle {
-        hook    = "prestart"
+        hook    = "poststart"
        sidecar = true
      }

@ -188,18 +180,15 @@ job "[[ .instance ]]" {
        pids_limit      = 300
      }

-      vault {
-        policies     = ["[[ .instance ]][[ .consul.suffix ]]"]
-        env          = false
-        disable_file = true
-      }
+[[ template "common/vault.policies" $c ]]

      env {
        PAPERLESS_MODE = "task-queue"
        TMPDIR         = "/alloc/tmp"
      }

-[[ template "common/file_env" merge $c.env .paperless.env ]]
+[[ template "common/file_env" $c.env ]]
+[[ template "paperless-ngx/dbenv" $c ]]

      volume_mount {
        volume      = "data"
--- a/templates/dbenv
+++ b/templates/dbenv
@ -0,0 +1,21 @@
+[[- /*
+vim: syntax=hcl
+*/]]
+      template {
+        data =<<_EOT
+[[- if ne .postgres.pooler.engine "none" ]]
+PAPERLESS_DBHOST=127.0.0.1
+PAPERLESS_DBPORT=6432
+PAPERLESS_DBUSER=[[ .instance ]]
+PAPERLESS_DBPASS={{ env "NOMAD_ALLOC_ID" }}
+[[- else ]]
+PAPERLESS_DBHOST=[[ .postgres.host ]]
+PAPERLESS_DBPORT=[[ .postgres.port ]]
+PAPERLESS_DBUSER=[[ .postgres.user ]]
+PAPERLESS_DBPASS=[[ .postgres.password ]]
+[[- end ]]
+_EOT
+        destination = "secrets/.db.env"
+        perms = 400
+        env = true
+      }
--- a/variables.yml
+++ b/variables.yml
@ -12,13 +12,23 @@ paperless:
        - destination_name: postgres[[ .consul.suffix ]]
          local_bind_port: 5432

+  # Vault policies to add to the containers
+  vault:
+    policies:
+      - '[[ .instance ]][[ .consul.suffix ]]'
+
+  # Parameters for the postgres database
+  postgres:
+    host: 127.0.0.1
+    port: 5432
+    database: '[[ .instance ]]'
+    user: '{{ with secret "[[ .vault.prefix ]]database/creds/[[ .instance ]]" }}{{ .Data.username }}{{ end }}'
+    password: '{{ with secret "[[ .vault.prefix ]]database/creds/[[ .instance ]]" }}{{ .Data.password }}{{ end }}'
+
  # Env var to set in the containers
  # The ones here will be inherited by all containers
  env:
-    PAPERLESS_DBUSER: '{{ with secret "[[ .vault.prefix ]]database/creds/[[ .instance ]]" }}{{ .Data.username }}{{ end }}'
-    PAPERLESS_DBPASS: '{{ with secret "[[ .vault.prefix ]]database/creds/[[ .instance ]]" }}{{ .Data.password }}{{ end }}'
    PAPERLESS_SECRET_KEY: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .instance ]]" }}{{ .Data.data.secret_key }}{{ end }}'
-    PAPERLESS_DBNAME: '[[ .instance ]]'
    PAPERLESS_CORS_ALLOWED_HOSTS: '[[ .paperless.webserver.public_url ]]'
    PAPERLESS_URL: '[[ .paperless.webserver.public_url ]]'
    PAPERLESS_CONVERT_TMPDIR: /alloc/data