Start working on MariaDB

This commit is contained in:
Daniel Berteaud 2023-10-25 23:00:31 +02:00
parent bfecbf612e
commit 75db82f0db
16 changed files with 573 additions and 0 deletions

4
bundles.yml Normal file
View File

@ -0,0 +1,4 @@
---
dependencies:
- url: ../common.git

View File

@ -0,0 +1,12 @@
Kind = "service-intentions"
Name = "[[ .mariadb.instance ]][[ .consul.suffix ]]"
Sources = [
{
Name = "[[ (merge .mariadb.server.traefik .traefik).instance ]]"
Action = "allow"
},
{
Name = "[[ .mariadb.instance ]]-manage[[ .consul.suffix ]]"
Action = "allow"
}
]

View File

@ -0,0 +1,6 @@
FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]]
MAINTAINER [[ .docker.maintainer ]]
RUN set -eux &&\
apk --no-cache upgrade &&\
apk --no-cache add mariadb-client

20
images/mariadb/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
FROM [[ .mariadb.manage.image ]]
MAINTAINER [[ .docker.maintainer ]]
ENV MYSQL_CONF_10_section=mysqld \
MYSQL_CONF_11_innodb_buffer_pool_size=50%
RUN set -eux &&\
apk --no-cache upgrade &&\
apk --no-cache add mariadb mariadb-server-utils &&\
chown mysql:mysql /etc/my.cnf.d &&\
rm -f /etc/my.cnf.d/* &&\
mkdir /data /run/mysqld &&\
chown mysql:mysql /data /run/mysqld &&\
chmod 700 /data
COPY root/ /
EXPOSE 3306
USER mysql
CMD ["mariadbd", "--console", "--skip-name-resolve"]

View File

@ -0,0 +1,44 @@
#!/bin/sh
set -eo pipefail
get_max_mem(){
if [ -e /sys/fs/cgroup/memory.max ]; then
# Read /sys/fs/cgroup/memory.max
MAX=$(cat /sys/fs/cgroup/memory.max)
# If it's "max", then the container has no limit, and we must detect the available RAM
if [ "${MAX}" = "max" ]; then
echo $(($(cat /proc/meminfo | grep MemTotal | sed -E 's/MemTotal:\s+([0-9]+)\s+kB/\1/')/1024))
else
echo $(($(cat /sys/fs/cgroup/memory.max)/1024/1024))
fi
else
echo $(($(cat /proc/meminfo | grep MemTotal | sed -E 's/MemTotal:\s+([0-9]+)\s+kB/\1/')/1024))
fi
}
if mount | grep -q ' /etc/my.cnf '; then
echo "/etc/my.cnf is mounted, skiping config from env vars"
else
echo "Configuring from env vars"
for VAR in $(printenv | grep -E '^MYSQL_CONF_' | sed -E 's/MYSQL_CONF_([^=]+)=.*/\1/' | sort -V); do
DIRECTIVE=$(echo ${VAR} | sed -E 's/^[0-9]+_//')
VALUE=$(printenv MYSQL_CONF_${VAR})
if [ "${DIRECTIVE}" = "section" ]; then
echo "[${VALUE}]" >> /etc/my.cnf.d/env.cnf
else
# Allow some memory related settings to be expressed as a %
if echo ${DIRECTIVE} | grep -q -E "^(innodb_buffer_pool_size)$"; then
if echo ${VALUE} | grep -q -E "[0-9]+%$"; then
PERCENT=$(echo $VALUE | sed -E 's|%$||')
MAX_MEM=$(get_max_mem)
VALUE=$((${MAX_MEM}*${PERCENT}/100))MB
fi
fi
echo "Adding ${DIRECTIVE} = ${VALUE} in /etc/my.cnf.d/env.cnf"
echo "${DIRECTIVE} = ${VALUE}" >> /etc/my.cnf.d/env.cnf
fi
done
fi

View File

@ -0,0 +1,36 @@
#!/bin/sh
set -euo pipefail
mkdir -p /data/db
if [ -d /data/db/mysql ]; then
echo "MariaDB is already initialized"
else
echo "Bootstraping MariaDB"
mysql_install_db
MYSQL_DATABASE=${MYSQL_DATABASE:-""}
MYSQL_USER=${MYSQL_USER:-""}
MYSQL_PASSWORD=${MYSQL_PASSWORD:-""}
cat << EOF > /tmp/mariainit.sql
USE mysql;
FLUSH PRIVILEGES;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' identified by '$MYSQL_ROOT_PASSWORD' WITH GRANT OPTION;
DROP DATABASE test;
EOF
if [ "$MYSQL_DATABASE" != "" ]; then
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` CHARACTER SET utf8 COLLATE utf8_general_ci;" >> $tfile
if [ "$MYSQL_USER" != "" ]; then
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* to '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';" >> $tfile
fi
fi
mariadbd --bootstrap --verbose=0 --skip-name-resolve < /tmp/mariainit.sql
rm -f /tmp/mariainit.sql
fi

View File

@ -0,0 +1,15 @@
[client-server]
port = 3306
socket = /run/mysqld/mysqld.sock
[mysqld]
pid-file = /tmp/mysql.pid
symbolic-links = 0
datadir = /data/db
tmpdir = /tmp
default_storage_engine = InnoDB
innodb_file_per_table = 1
innodb_log_file_size = 512M
!includedir /etc/my.cnf.d

19
init/vault-database Executable file
View File

@ -0,0 +1,19 @@
#!/bin/sh
set -euo pipefail
if [ "$(vault secrets list -format json | jq -r '.["[[ .vault.prefix ]]database/"].type')" != "database" ]; then
vault secrets enable -path [[ .vault.prefix ]]database database
fi
if [ "$(vault list -format json [[ .vault.prefix ]]database/config | jq '.[] | test("^[[ .mariadb.instance ]]$")')" = "false" ]; then
vault write [[ .vault.prefix ]]database/config/[[ .mariadb.instance ]] \
plugin_name="mysql-database-plugin" \
connection_url="{{username}}:{{password}}@tcp([[ (urlParse .mariadb.server.public_address).Host ]])/" \
allowed_roles="*" \
username=vault \
password="$(vault kv get -field vault_initial_pwd [[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]])" \
disable_escaping=true
vault write -force [[ .vault.prefix ]]database/rotate-root/[[ .mariadb.instance ]]
fi

118
manage.nomad.hcl Normal file
View File

@ -0,0 +1,118 @@
[[ $c := merge .mariadb.manage . -]]
job "[[ .mariadb.instance ]]-manage" {
type = "batch"
[[ template "common/job_start.tpl" $c ]]
meta {
# Force job to run each time
run = "${uuidv4()}"
}
group "manage" {
network {
mode = "bridge"
}
service {
name = "[[ .mariadb.instance ]]-manage[[ $c.consul.suffix ]]"
[[ template "common/connect.tpl" $c ]]
}
[[ template "common/task.wait_for.tpl" dict
"ctx" .
"wait_for" (coll.Slice (dict "service" .mariadb.instance)) ]]
task "manage" {
driver = [[ $c.nomad.driver | toJSON ]]
config {
image = [[ .mariadb.manage.image | toJSON ]]
pids_limit = 50
readonly_rootfs = true
command = "/local/manage.sh"
volumes = [
"secrets/my.cnf:/root/.my.cnf:ro"
]
}
vault {
policies = ["[[ .mariadb.instance ]][[ $c.consul.suffix ]]"]
}
env {
[[ template "common/env.tpl" $c.env ]]
}
template {
data = <<_EOT
[[- range $idx, $db := .mariadb.manage.databases ]]
MY_DB_[[ $idx ]]=[[ $db.name ]]
[[- if has $db "charset" ]]
MY_DB_[[ $idx ]]_CHARSET=[[ $db.charset ]]
[[- end ]]
[[- if has $db "collate" ]]
MY_DB_[[ $idx ]]_COLLATE=[[ $db.collate ]]
[[- end ]]
[[- end ]]
[[- range $idx, $user := .mariadb.manage.users ]]
MY_USER_[[ $idx ]]=[[ $user.name ]]
[[- if has $user "host" ]]
MY_USER_[[ $idx ]]_HOST=[[ $user.host ]]
[[- else ]]
MY_USER_[[ $idx ]]_HOST=%
[[- end ]]
[[- if has $user "password" ]]
MY_USER_[[ $idx ]]_PASSWORD=[[ $user.password ]]
[[- end ]]
[[- if has $user "grants" ]]
[[- range $gidx, $grant := $user.grants ]]
MY_USER_[[ $idx ]]_GRANT_[[ $gidx ]]=[[ $grant ]]
[[- end ]]
[[- end ]]
[[- end ]]
_EOT
destination = "secrets/userdb.env"
uid = 100000
gid = 100000
perms = 0400
env = true
}
template {
data = <<_EOT
[[ template "mariadb/manage.sh.tpl" $c ]]
_EOT
destination = "local/manage.sh"
uid = 100000
gid = 100000
perms = 755
}
template {
data = <<_EOT
[client]
host = 127.0.0.1
user = root
password = {{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}
_EOT
destination = "secrets/my.cnf"
uid = 100100
gid = 100101
perms = 640
}
template {
data = <<_EOT
VAULT_INITIAL_PASSWORD={{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.vault_initial_pwd }}{{ end }}
_EOT
destination = "secrets/manage.env"
uid = 100000
gid = 100000
perms = 400
env = true
}
[[ template "common/resources.tpl" .mariadb.manage.resources ]]
}
}
}

160
mariadb.nomad.hcl Normal file
View File

@ -0,0 +1,160 @@
[[- $c := merge .mariadb.server . -]]
job [[ .mariadb.instance | toJSON ]] {
[[ template "common/job_start.tpl" $c ]]
group "server" {
network {
mode = "bridge"
}
volume "mariadb" {
type = [[ .mariadb.server.volumes.mariadb.type | toJSON ]]
source = [[ .mariadb.server.volumes.mariadb.source | toJSON ]]
access_mode = "single-node-writer"
attachment_mode = "file-system"
per_alloc = true
}
service {
name = "[[ .mariadb.instance ]][[ $c.consul.suffix ]]"
port = 3306
[[ template "common/connect.tpl" $c ]]
check {
name = "alive"
type = "script"
task = "mariadb"
command = "mysqladmin"
args = [
"ping"
]
timeout = "10s"
interval = "5s"
}
[[- if $c.traefik.enabled ]]
tags = [
"[[ $c.traefik.instance ]].enable=true",
"[[ $c.traefik.instance ]].tcp.routers.[[ .mariadb.instance ]][[ $c.consul.suffix ]].rule=HostSNI(`*`)",
"[[ $c.traefik.instance ]].tcp.routers.[[ .mariadb.instance ]][[ $c.consul.suffix ]].entrypoints=[[ join $c.traefik.entrypoints "," ]]",
"[[ $c.traefik.instance ]].tcp.routers.[[ .mariadb.instance ]][[ $c.consul.suffix ]].middlewares=[[ join $c.traefik.middlewares "," ]]"
]
[[- end ]]
}
# Run mysql_upgrade
task "manage" {
driver = [[ $c.nomad.driver | toJSON ]]
lifecycle {
hook = "poststart"
}
config {
image = [[ .mariadb.manage.image | toJSON ]]
pids_limit = 50
readonly_rootfs = true
command = "/local/mysql_upgrade.sh"
volumes = [
"secrets/my.cnf:/root/.my.cnf:ro"
]
}
vault {
policies = ["[[ .mariadb.instance ]][[ .consul.suffix ]]"]
env = false
disable_file = true
}
template {
data = <<_EOT
[client]
user = root
password = {{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}
_EOT
destination = "secrets/my.cnf"
uid = 100100
gid = 100101
perms = 640
}
template {
data = <<_EOT
[[ template "mariadb/mysql_upgrade.sh.tpl" $c ]]
_EOT
destination = "local/mysql_upgrade.sh"
perms = 755
}
resources {
cpu = 10
memory = 32
memory_max = 64
}
}
task "mariadb" {
driver = [[ $c.nomad.driver | toJSON ]]
leader = true
kill_timeout = "5m"
config {
image = [[ .mariadb.server.image | toJSON ]]
volumes = [
"secrets/:/etc/my.cnf.d",
"secrets/my.conf:/var/lib/mysql/.my.cnf:ro",
]
pids_limit = 300
#readonly_rootfs = true
}
vault {
policies = ["[[ .mariadb.instance ]][[ .consul.suffix ]]"]
env = false
disable_file = true
}
env {
MYSQL_CONF_11_bind-address = "127.0.0.1"
[[ template "common/env.tpl" $c.env ]]
}
template {
data = <<_EOT
{{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}
MYSQL_ROOT_PASSWORD={{ .Data.data.root_pwd }}
{{ end }}
_EOT
destination = "secrets/mariadb.env"
uid = 100000
gid = 100000
perms = 400
env = true
}
template {
data = <<_EOT
[client]
user = root
password = {{ with secret "[[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}
_EOT
destination = "secrets/my.conf"
uid = 100100
gid = 100101
perms = 640
}
volume_mount {
volume = "mariadb"
destination = "/data"
}
[[ template "common/resources.tpl" .mariadb.server.resources ]]
}
}
}

16
prep.d/10-rand-pwd.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/sh
set -euo pipefail
if ! vault kv list [[ .vault.prefix ]]kv/service 2>/dev/null | grep -q -E '^[[ .mariadb.instance ]]$'; then
vault kv put [[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]] \
root_pwd=$(pwgen -s -n 50 1) \
vault_initial_pwd=$(pwgen -s -n 50 1)
fi
for PWD in root_pwd vault_initial_pwd; do
if ! vault kv get -field ${PWD} [[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]] >/dev/null 2>&1; then
vault kv patch [[ .vault.prefix ]]kv/service/[[ .mariadb.instance ]] \
${PWD}=$(pwgen -s -n 50 1)
fi
done

1
prep.d/mv_conf.sh Executable file
View File

@ -0,0 +1 @@
[[ template "common/mv_conf.sh.tpl" dict "ctx" . "services" (dict "mariadb" .mariadb.instance) ]]

47
templates/manage.sh.tpl Normal file
View File

@ -0,0 +1,47 @@
#!/bin/sh
# vim: syntax=sh
set -euo pipefail
echo "Creating vault user"
mysql <<_EOSQL
CREATE USER IF NOT EXISTS 'vault'@'%' IDENTIFIED BY '${VAULT_INITIAL_PASSWORD}';
GRANT ALL PRIVILEGES ON *.* TO 'vault'@'%' WITH GRANT OPTION;
_EOSQL
echo "Create databases"
for IDX in $(printenv | grep -E '^MY_DB_([0-9]+)=' | sed -E 's/^MY_DB_([0-9]+)=.*/\1/'); do
DB_NAME=$(printenv MY_DB_${IDX})
echo "Found DB ${DB_NAME} to create"
DB_CHARSET=$(printenv MY_DB_${IDX}_CHARSET || echo "utf8mb4")
DB_COLLATE=$(printenv MY_DB_${IDX}_COLLATE || echo "utf8mb4_general_ci")
echo "Create database ${DB_NAME} (CHARACTER SET \"${DB_CHARSET}\" COLLATE \"${DB_COLLATE}\") if needed"
mysql <<_EOSQL
CREATE DATABASE IF NOT EXISTS ${DB_NAME} CHARACTER SET "${DB_CHARSET}" COLLATE "${DB_COLLATE}"
_EOSQL
done
echo "Creating users"
for IDX in $(printenv | grep -E '^MY_USER_([0-9]+)=' | sed -E 's/^MY_USER_([0-9]+)=.*/\1/'); do
DB_USER=$(printenv MY_USER_${IDX})
echo "Found DB User ${DB_USER} to create"
DB_HOST=$(printenv MY_USER_${IDX}_HOST || echo '%')
DB_PASSWORD=$(printenv MY_USER_${IDX}_PASSWORD || echo '')
if [ "${DB_PASSWORD}" = "" ]; then
mysql <<_EOSQL
CREATE USER IF NOT EXISTS '${DB_USER}'@'${DB_HOST}';
_EOSQL
else
mysql <<_EOSQL
CREATE USER IF NOT EXISTS '${DB_USER}'@'${DB_HOST}' IDENTIFIED BY '${DB_PASSWORD}';
_EOSQL
fi
echo "Applying grants for ${DB_USER}"
for GRANT in $(printenv | grep -E "^MY_USER_${IDX}_GRANT_([0-9]+)=)" | sed -E "s/^MY_USER_${IDX}_GRANT_([0-9]+)=.*/\1/"); do
mysql <<_EOSQL
GRANT $(printenv MY_USER_${IDX}_GRANT_${GRANT});
_EOSQL
done
done

View File

@ -0,0 +1,16 @@
#!/bin/sh
set -euo pipefail
COUNT=0
while true; do
if mysqladmin ping; then
echo "MariaDB is ready, running mysql_upgrade"
mysql_upgrade
exit 0
fi
echo "MariaDB not ready yet, waiting a bit more"
COUNT=$((COUNT+1))
sleep 1
done

56
variables.yml Normal file
View File

@ -0,0 +1,56 @@
---
mariadb:
instance: mariadb
server:
image: danielberteaud/mariadb:latest
resources:
cpu: 100
memory: 1024
env: {}
public_address: mysql://mariadb.example.org:3306
traefik:
enabled: false
entrypoints:
- mariadb
consul:
connect:
disable_default_tcp_check: true
volumes:
mariadb:
type: csi
source: mariadb
manage:
image: danielberteaud/mariadb-client:latest
resources:
cpu: 10
memory: 10
memory_max: 50
env: {}
databases: []
users: []
# users:
# - name: myuser
# host: %
# password: p@ssw0rd
# grants:
# - SELECT ON kimai.*
# - INSERT,DELETE,UPDATE ON bookstack.*
consul:
connect:
upstreams:
- destination_name: '[[ .mariadb.instance ]][[ .consul.suffix ]]'
local_bind_port: 3306

View File

@ -0,0 +1,3 @@
path "[[ .vault.prefix ]]kv/data/service/[[ .mariadb.instance ]]" {
capabilities = ["read"]
}