More work, including cron based backups

consul/config/service-intentions/mariadb.hcl

@@ -8,5 +8,9 @@ Sources = [
   {
     Name = "[[ .mariadb.instance ]]-manage[[ .consul.suffix ]]"
     Action = "allow"
+  },
+  {
+    Name = "[[ .mariadb.instance ]]-backup[[ .consul.suffix ]]"
+    Action = "allow"
   }
 ]

images/mariadb-backup/Dockerfile

@@ -0,0 +1,7 @@
+FROM [[ .docker.repo ]][[ .docker.base_images.mariadb_client.image ]]
+MAINTAINER [[ .docker.maintainer ]]
+
+RUN set -eux &&\
+    apk --no-cache update &&\
+    apk --no-cache add supercronic
+

images/mariadb-client/Dockerfile

@@ -1,6 +0,0 @@
-FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]]
-MAINTAINER [[ .docker.maintainer ]]
-
-RUN set -eux &&\
-    apk --no-cache upgrade &&\
-    apk --no-cache add mariadb-client mariadb-server-utils

images/mariadb/Dockerfile

@@ -1,4 +1,4 @@
-FROM [[ .mariadb.manage.image ]]
+FROM [[ .docker.repo ]][[ .docker.base_images.mariadb_client.image ]]
 MAINTAINER [[ .docker.maintainer ]]
 
 ENV MYSQL_CONF_10_section=mysqld \

manage.nomad.hcl

@@ -54,8 +54,9 @@ MY_DB_[[ $idx ]]_CHARSET=[[ $db.charset ]]
 MY_DB_[[ $idx ]]_COLLATE=[[ $db.collate ]]
   [[- end ]]
 [[- end ]]
-[[- range $idx, $user := .mariadb.manage.users ]]
-MY_USER_[[ $idx ]]=[[ $user.name ]]
+[[- $idx := 0 ]]
+[[- range $name, $user := .mariadb.manage.users ]]
+MY_USER_[[ $idx ]]=[[ $name ]]
   [[- if has $user "host" ]]
 MY_USER_[[ $idx ]]_HOST=[[ $user.host ]]
   [[- else ]]
@@ -69,6 +70,7 @@ MY_USER_[[ $idx ]]_PASSWORD=[[ $user.password ]]
 MY_USER_[[ $idx ]]_GRANT_[[ $gidx ]]=[[ $grant ]]
     [[- end ]]
   [[- end ]]
+  [[ $idx = add $idx 1 ]]
 [[- end ]]
 _EOT
         destination = "secrets/userdb.env"
@@ -103,7 +105,10 @@ _EOT
 
       template {
         data = <<_EOT
-VAULT_INITIAL_PASSWORD={{  with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.vault_initial_pwd }}{{ end }}
+{{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}
+VAULT_INITIAL_PASSWORD={{ .Data.data.vault_initial_pwd }}
+BACKUP_PASSWORD={{ .Data.data.backup_pwd }}
+{{ end }}
 _EOT
         destination = "secrets/manage.env"
         uid = 100000

mariadb.nomad.hcl

@@ -10,8 +10,8 @@ job [[ .mariadb.instance | toJSON ]] {
     }
 
     volume "mariadb" {
-      type            = [[ .mariadb.server.volumes.mariadb.type | toJSON ]]
-      source          = [[ .mariadb.server.volumes.mariadb.source | toJSON ]]
+      type            = [[ .mariadb.volumes.data.type | toJSON ]]
+      source          = [[ .mariadb.volumes.data.source | toJSON ]]
       access_mode     = "single-node-writer"
       attachment_mode = "file-system"
       per_alloc       = true
@@ -54,12 +54,9 @@ job [[ .mariadb.instance | toJSON ]] {
       }
 
       config {
-        image = [[ .mariadb.manage.image | toJSON ]]
+        image = [[ .mariadb.server.image | toJSON ]]
         pids_limit = 100
         command = "/local/mysql_upgrade.sh"
-        volumes = [
-          "secrets/my.cnf:/root/.my.cnf:ro"
-        ]
       }
 
       vault {
@@ -75,7 +72,7 @@ user = root
 host = 127.0.0.1
 password = {{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}
 _EOT
-        destination = "secrets/my.cnf"
+        destination = "secrets/.my.cnf"
         uid = 100100
         gid = 100101
         perms = 640
@@ -162,4 +159,86 @@ _EOT
 
     }
   }
+
+[[- if .mariadb.backup.dumps.enabled ]]
+[[- $c := merge .mariadb.backup . ]]
+
+  group "backup" {
+    network {
+      mode = "bridge"
+    }
+
+    volume "backup" {
+      type            = [[ .mariadb.volumes.backup.type | toJSON ]]
+      source          = [[ .mariadb.volumes.backup.source | toJSON ]]
+      access_mode     = "multi-node-multi-writer"
+      attachment_mode = "file-system"
+    }
+
+    service {
+      name = "[[ .mariadb.instance ]]-backup[[ $c.consul.suffix ]]"
+[[ template "common/connect.tpl" $c ]]
+    }
+
+[[ template "common/task.wait_for.tpl" dict 
+  "ctx" .
+  "wait_for" (coll.Slice (dict "service" .mariadb.instance)) ]]
+
+    task "backup" {
+      driver = [[ $c.nomad.driver | toJSON ]]
+
+      config {
+        image = [[ .mariadb.backup.image | toJSON ]]
+        pids_limit = 100
+        readonly_rootfs = true
+        command = "supercronic"
+        args = [
+          "/secrets/backup.cron"
+        ]
+      }
+
+      vault {
+        policies = ["[[ .mariadb.instance ]][[ $c.consul.suffix ]]"]
+        env = false
+        disable_file = true
+      }
+
+      template {
+        data = <<_EOT
+[client]
+user = root
+host = 127.0.0.1
+password = {{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}
+_EOT
+        destination = "secrets/.my.cnf"
+        uid = 100000
+        gid = 100000
+        perms = 400
+      }
+
+      template {
+        data =<<_EOT
+[[ template "mariadb/dump.sh.tpl" $c ]]
+_EOT
+        destination = "local/dump.sh"
+        perms = 755
+      }
+
+      template {
+        data =<<_EOT
+[[ template "mariadb/backup.cron.tpl" $c ]]
+_EOT
+        destination = "secrets/backup.cron"
+      }
+
+      volume_mount {
+        volume      = "backup"
+        destination = "/backup"
+      }
+
+[[ template "common/resources.tpl" .mariadb.server.resources ]]
+
+    }
+  }
+[[- end ]]
 }

templates/backup.cron.tpl

@@ -0,0 +1,3 @@
+[[- if and .mariadb.backup.dumps.enabled (not (eq .mariadb.backup.dumps.cron "")) ]]
+[[ .mariadb.backup.dumps.cron ]] /local/dump.sh
+[[- end ]]

templates/dump.sh.tpl

@@ -0,0 +1,17 @@
+#!/bin/sh
+  
+set -eo pipefail
+
+HOME=/secrets
+DEST=/backup
+
+export ZSTD_CLEVEL=[[ .mariadb.backup.dumps.compression ]]
+export ZSTD_NBTHREADS=0
+
+echo Removing previous dumps
+rm -f ${DEST}/*.sql.zst
+
+for DB in $(mysqlshow | awk '{print $2}' | grep -v Databases | grep -v -E '^(information_schema)$'); do
+  echo "Dumping ${DB} to ${DEST}/${DB}.sql.zst"
+  mysqldump --ignore-table=mysql.event --single-transaction --add-drop-table $DB | zstd --no-progress -o ${DEST}/${DB}.sql.zst
+done

templates/manage.sh.tpl

@@ -4,7 +4,7 @@
 
 set -euo pipefail
 
-echo "Creating vault user"
+echo "Creating vault and backup user"
 mysql <<_EOSQL
 CREATE USER IF NOT EXISTS 'vault'@'%' IDENTIFIED BY '${VAULT_INITIAL_PASSWORD}';
 GRANT ALL PRIVILEGES ON *.* TO 'vault'@'%' WITH GRANT OPTION;

templates/mysql_upgrade.sh.tpl

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+HOME=/secrets
+
 COUNT=0
 while true; do
   if mysqladmin ping; then

variables.yml

@@ -8,7 +8,7 @@ mariadb:
 
     resources:
       cpu: 100
-      memory: 1024
+      memory: 768
 
     env: {}
 
@@ -23,14 +23,9 @@ mariadb:
       connect:
         disable_default_tcp_check: true
 
-    volumes:
-      mariadb:
-        type: csi
-        source: mariadb
-
   manage:
 
-    image: danielberteaud/mariadb-client:latest
+    image: '[[ .docker.repo ]][[ .docker.base_images.mariadb_client.image ]]'
 
     resources:
       cpu: 10
@@ -40,17 +35,40 @@ mariadb:
     env: {}
 
     databases: []
-    users: []
-    # users:
-    #   - name: myuser
-    #     host: %
-    #     password: p@ssw0rd
-    #     grants:
-    #       - SELECT ON kimai.*
-    #       - INSERT,DELETE,UPDATE ON bookstack.*
+    users: {}
+    #  kimai:
+    #    host: %
+    #    password: '{{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.kimai_pwd }}{{ end }}'
+    #    grants:
+    #      - 'ALL PRIVILEGES ON kimai.*'
 
     consul:
       connect:
         upstreams:
           - destination_name: '[[ .mariadb.instance ]][[ .consul.suffix ]]'
             local_bind_port: 3306
+
+  backup:
+    image: danielberteaud/mariadb-backup:latest
+    resources:
+      cpu: 200
+      memory: 128
+
+    consul:
+      connect:
+        upstreams:
+          - destination_name: '[[ .mariadb.instance ]][[ .consul.suffix ]]'
+            local_bind_port: 3306
+
+    dumps:
+      enabled: False
+      compression: 6
+      cron: 12 03 * * *
+
+  volumes:
+    data:
+      type: csi
+      source: mariadb-data
+    backup:
+      type: csi
+      source: mariadb-backup