Initial commit, supporting single or multi nodes

bundles.yml

@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - url: ../common.git

consul/config/service-defaults/minio.hcl

@@ -0,0 +1,3 @@
+Kind = "service-defaults"
+Name = "[[ .instance ]][[ .consul.suffix ]]"
+Protocol = "http"

consul/config/service-intentions/minio.hcl

@@ -0,0 +1,26 @@
+Kind = "service-intentions"
+Name = "[[ .instance ]][[ .consul.suffix ]]"
+Sources = [
+  {
+    Name = "[[ (merge .minio.server.api .).traefik.instance ]]"
+    Permissions = [
+      {
+        Action = "allow"
+        HTTP {
+          PathPrefix = "/"
+        }
+      }
+    ]
+  },
+  {
+    Name = "prometheus[[ .consul.suffix ]]"
+    Permissions = [
+      {
+        Action = "allow"
+        HTTP {
+          PathPrefix = "/minio/v2/metrics"
+        }
+      }
+    ]
+  }
+]

images/minio/Dockerfile

@@ -0,0 +1,29 @@
+FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]]
+MAINTAINER [[ .docker.maintainer ]]
+
+ARG MINIO_VERSION=[[ .minio.server.version ]]
+
+ADD --chown=root:root --chmod=755 https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.${MINIO_VERSION} /usr/local/bin/minio
+
+RUN set -euxo pipefail &&\
+    apk --no-cache update &&\
+    apk --no-cache add \
+      openssl \
+      curl \
+    &&\
+    cd /usr/local/bin &&\
+    curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.${MINIO_VERSION}.sha256sum |\
+      sed -e "s/minio\.RELEASE\.${MINIO_VERSION}/minio/g" |\
+      sha256sum -c &&\
+    addgroup -g 9000 minio &&\
+    adduser --system \
+            --ingroup minio \
+            --disabled-password \
+            --uid 9000 \
+            --shell /sbin/nologin \
+            minio
+
+COPY root/ /
+
+USER minio
+CMD ["minio"]

images/minio/root/entrypoint.d/10-split-cert

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -eo pipefail
+
+# This snippet can split a PEM bundle into a public.crt and a private.key files for minio to use
+
+if [ -n "${MINIO_CERT_BUNDLE}" -a -e "${MINIO_CERT_BUNDLE}" -a -n "${MINIO_CERTS_DIR}" ]; then
+  mkdir -p ${MINIO_CERTS_DIR}/CAs
+  openssl x509 -in "${MINIO_CERT_BUNDLE}" -out ${MINIO_CERTS_DIR}/public.crt
+  openssl pkey -in "${MINIO_CERT_BUNDLE}" -out ${MINIO_CERTS_DIR}/private.key
+  chmod 600 ${MINIO_CERTS_DIR}/private.key
+  if [ -n "${MINIO_CA}" -a -e "${MINIO_CA}" ]; then
+    cp "${MINIO_CA}" ${MINIO_CERTS_DIR}/CAs/
+  fi
+fi

init/minio-pki

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+[[ $c := merge .minio . ]]
+[[ template "common/vault.mkpki.sh.tpl" $c ]]
+
+vault write [[ $c.vault.pki.path ]]/roles/minio \
+  allowed_domains="[[ .instance ]][[ .consul.suffix ]].service.[[ .consul.domain ]],[[ .instance ]]-cluster[[ .consul.suffix ]].service.[[ .consul.domain ]]" \
+  allow_bare_domains=true \
+  allow_subdomains=true \
+  allow_localhost=false \
+  allow_ip_sans=true \
+  allow_wildcard_certificates=false \
+  max_ttl=72h

minio.nomad.hcl

@@ -0,0 +1,179 @@
+job "[[ .instance ]]" {
+
+[[- $c := merge .minio . ]]
+[[ template "common/job_start" $c ]]
+
+  group "minio" {
+[[- $c := merge .minio.server $c ]]
+[[ template "common/group_start" $c ]]
+[[ template "common/volumes" $c ]]
+
+    network {
+      mode = "bridge"
+[[- if conv.ToBool $c.prometheus.enabled ]]
+      port "metrics" {}
+[[- end ]]
+[[- /* When running in distributed mode, every node must be reachable on the same port , so assign a static one*/]]
+[[- if gt $c.count 1 ]]
+      port "api" {
+        static = [[ $c.api.port ]]
+      }
+[[- end ]]
+
+      # Ensure local minio instance can identify itself
+      hostname = "minio-${NOMAD_ALLOC_INDEX}.[[ .instance ]]-cluster[[ .consul.suffix ]].service.[[ .consul.domain ]]"
+    }
+
+    # This is the main service, used to reach both the console and the S3 API
+    # It'll usually be exposed by Traefik. This service will appear as passing onlt when the corresponding
+    # minio is ready, so no requests are routed until they can be processed
+    service {
+      name = "[[ .instance ]][[ .consul.suffix ]]"
+      port = 8000
+[[ template "common/service_meta" $c ]]
+[[ template "common/connect" $c ]]
+
+      check {
+        type   = "http"
+        path   = "/minio/health/live"
+        expose = true
+[[ template "common/check_settings" $c ]]
+      }
+
+      tags = [
+        "minio-${NOMAD_ALLOC_INDEX}",
+[[ template "common/traefik_tags" merge $c.api $c ]]
+[[ template "common/traefik_tags" merge $c.console $c ]]
+[[ template "common/traefik_tags" merge $c.metrics $c ]]
+      ]
+    }
+
+[[- if gt $c.count 1 ]]
+
+    # A dummy service for minio instances to find each others. We define no check at all
+    # so the corresponding DNS entry are immediatly published in Consul
+    service {
+      name = "[[ .instance ]]-cluster[[ .consul.suffix ]]"
+      port = "api"
+
+      tags = [
+        "minio-${NOMAD_ALLOC_INDEX}",
+      ]
+    }
+[[- end ]]
+
+    # A small nginx proxy, used to pultiplexe S3 API and console on a single port
+    # We could instead use two connect service (so 2 envoy sidecars), but nginx is lighter
+    # It's also used to expose the service as plain http for the service mesh
+    task "nginx" {
+      driver = "[[ $c.nomad.driver ]]"
+
+      lifecycle {
+        hook    = "prestart"
+        sidecar = true
+      }
+
+      config {
+        image           = "nginxinc/nginx-unprivileged:alpine"
+        readonly_rootfs = true
+        pids_limit      = 100
+[[ template "common/tmpfs" "/tmp" ]]
+        volumes         = ["local/nginx.conf:/etc/nginx/conf.d/default.conf:ro"]
+      }
+
+      template {
+        data = <<_EOT
+[[ template "minio/nginx.conf" $c ]]
+_EOT
+        destination = "local/nginx.conf"
+      }
+
+      resources {
+        cpu        = 20
+        memory     = 20
+        memory_max = 56
+      }
+    }
+
+    task "minio" {
+      driver       = "[[ $c.nomad.driver ]]"
+      leader       = true
+      # Give minio some time to shutdown
+      kill_timeout = "10m"
+
+      config {
+[[ template "common/image" $c ]]
+        command = "minio"
+        args = [
+          "server",
+          "--console-address=127.0.0.1:9001",
+[[- if gt $c.count 1 ]]
+          "--address=0.0.0.0:${NOMAD_ALLOC_PORT_api}",
+          "--certs-dir=/secrets/tls",
+[[- else ]]
+          "--address=127.0.0.1:9000",
+[[- end ]]
+        ]
+        pids_limit = "1000"
+      }
+
+      env {
+        MINIO_BROWSER_REDIRECT_URL = "[[ $c.console.public_url ]]"
+        # Always hit the local minio to handle S3 API calls from the console
+        MINIO_SERVER_URL           = "http://127.0.0.1:8000"
+[[- if not (has $c.env "MINIO_VOLUMES") ]]
+        MINIO_VOLUMES              = "[[- if gt $c.count 1 -]]https://minio-{0...[[ sub $c.count 1 ]]}.[[ .instance ]]-cluster[[ .consul.suffix ]].service.[[ .consul.domain ]]:[[ $c.api_port ]][[ end ]][[ $c.volumes.data.destination ]]/storage"
+[[- end ]]
+        MINIO_PROMETHEUS_AUTH_TYPE = "public"
+[[- if gt $c.count 1 ]]
+        MINIO_CA                   = "/local/minio.ca.pem"
+        MINIO_CERT_BUNDLE          = "/secrets/minio.bundle.pem"
+        MINIO_CERTS_DIR            = "/secrets/tls"
+[[- end ]]
+      }
+
+[[ template "common/file_env" $c ]]
+[[ template "common/artifacts" $c ]]
+[[ template "common/vault.policies" $c ]]
+
+[[- if gt $c.count 1 ]]
+      template {
+        data = <<_EOT
+{{- with pkiCert "[[ $.vault.pki.path ]]/issue/minio"
+                 (printf "common_name=minio-%s.[[ $.instance ]]-cluster[[ $.consul.suffix ]].service.[[ $.consul.domain ]]" (env "NOMAD_ALLOC_INDEX"))
+                 (printf "alt_name=minio-%s.[[ $.instance ]][[ $.consul.suffix ]].service.[[ $.consul.domain ]],[[ $.instance ]][[ $.consul.suffix ]].service.[[ $.consul.domain ]]" (env "NOMAD_ALLOC_INDEX"))
+                 (printf "ip_sans=%s" (env "NOMAD_HOST_IP_api"))
+                 (printf "ttl=%dh" (env "NOMAD_ALLOC_INDEX" | parseInt | multiply 24 | add 72)) }}
+{{ .Cert }}
+{{ .Key }}
+{{- end -}}
+_EOT
+        destination = "/secrets/minio.bundle.pem"
+        uid         = 100100
+        gid         = 100101
+        perms       = 400
+        change_mode = "script"
+        change_script {
+          command = "/entrypoint.d/10-split-cert"
+        }
+      }
+
+      # CA certificate chains
+      template {
+        data          = <<_EOT
+{{- with secret "[[ $c.vault.pki.path ]]/cert/ca_chain" }}
+{{ .Data.ca_chain }}
+{{- end }}
+_EOT
+        destination   = "local/minio.ca.pem"
+        change_script {
+          command = "/entrypoint.d/10-split-cert"
+        }
+      }
+[[- end ]]
+
+[[ template "common/volumes_mount" $c ]]
+[[ template "common/resources" $c ]]
+    }
+  }
+}

prep.d/minio-rand-secrets

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -euo pipefail
+
+[[ template "common/vault.rand_secrets" merge .minio . ]]

templates/nginx.conf

@@ -0,0 +1,105 @@
+[[- $proto := "http" ]]
+[[- $port := "9000" ]]
+[[- if gt .count 1 ]]
+  [[- $proto = "https" ]]
+  [[- $port = "{{ env \"NOMAD_ALLOC_PORT_api\" }}" ]]
+[[- end ]]
+
+# S3 API proxy
+server {
+  listen 127.0.0.1:8000 default;
+  server_name _;
+  server_tokens off;
+  root /usr/share/html;
+
+  set_real_ip_from 127.0.0.1;
+  real_ip_header X-Forwarded-For;
+  real_ip_recursive on;
+
+  ignore_invalid_headers off;
+  client_max_body_size 0;
+  proxy_buffering off;
+  proxy_request_buffering off;
+  proxy_set_header Host $http_host;
+  proxy_set_header X-Real-IP $remote_addr;
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  proxy_set_header X-Forwarded-Proto $scheme;
+  proxy_http_version 1.1;
+  #proxy_set_header Connection "";
+  chunked_transfer_encoding off;
+
+  location / {
+    proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
+  }
+}
+
+# Console proxy
+server {
+  listen 127.0.0.1:8000;
+  server_name [[ (urlParse .console.public_url).Hostname ]];
+  server_tokens off;
+  root /usr/share/html;
+
+  set_real_ip_from 127.0.0.1;
+  real_ip_header X-Forwarded-For;
+  real_ip_recursive on;
+
+  ignore_invalid_headers off;
+  client_max_body_size 0;
+  proxy_buffering off;
+  proxy_request_buffering off;
+  chunked_transfer_encoding off;
+
+[[- if eq (.console.public_url | regexp.Replace "/$" "") (.api.public_url | regexp.Replace "/$" "") ]]
+  [[- fail "Console and API must use a different host, or path" ]]
+[[- end ]]
+[[- if and (eq ((urlParse .console.public_url).Hostname) ((urlParse .api.public_url).Hostname)) (eq ((urlParse .console.public_url).Path | regexp.Replace "/$" "" ) "") ]]
+  [[- fail "When sharing the same domain for console and S3 API, console must use a sub-path" ]]
+[[- end ]]
+
+[[- $console := "" ]]
+[[- if eq ((urlParse .console.public_url).Path) "" ]]
+  [[- $console = "/" ]]
+[[- else ]]
+  [[- $console = printf "%s/" ((urlParse .console.public_url).Path | regexp.Replace "/$" "") ]]
+[[- end ]]
+
+  location [[ $console ]]ws/ {
+[[- if ne $console "/" ]]
+    rewrite ^[[ $console ]]ws/(.*) /ws/$1 break;
+[[- end ]]
+
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "Upgrade";
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-NginX-Proxy true;
+    proxy_http_version 1.1;
+    proxy_pass [[ $proto ]]://127.0.0.1:9001;
+  }
+
+
+  location [[ $console ]] {
+[[- if ne $console "/" ]]
+    rewrite ^[[ $console ]](.*) /$1 break;
+[[- end ]]
+
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-NginX-Proxy true;
+    proxy_http_version 1.1;
+    proxy_pass [[ $proto ]]://127.0.0.1:9001;
+  }
+
+[[- if eq ((urlParse .console.public_url).Hostname) ((urlParse .api.public_url).Hostname) ]]
+
+  # Proxy the S3 API as it shares the same vhost as the console
+  location / {
+    proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
+  }
+[[- end ]]
+}

variables.yml

@@ -0,0 +1,87 @@
+---
+
+# Name of the instance
+instance: minio
+
+
+vault:
+
+  # Vault PKI used for minio
+  pki:
+    path: '[[ .vault.root ]]pki/[[ .instance ]]'
+    ou: S3 Storage
+
+  # Generate some random secrets
+  rand_secrets:
+    fields:
+      - root_pwd
+
+minio:
+
+  server:
+    # Number of nodes of the cluster. Note : this can only be set on cluster initialization
+    # Once initialized, you cannot change this number (up or down)
+    count: 1
+
+    # Version of minio
+    version: 2024-06-06T09-36-42Z
+  
+    # Docker image to use
+    image: '[[ .docker.repo ]]minio:[[ .minio.server.version ]]-2'
+  
+    # Resource allocation
+    resources:
+      cpu: 200
+      memory: 1024
+      memory_max: 1200
+  
+    # Custom env vars to set in the containers
+    env:
+      MINIO_ROOT_USER: '[[ .instance ]]'
+      MINIO_ROOT_PASSWORD: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.root_pwd }}{{ end }}'
+      MINIO_COMPRESSION_ENABLE: on
+  
+      MINIO_API_OBJECT_MAX_VERSIONS: 1000
+  
+    vault:
+      policies:
+        - '[[ .instance ]][[ .consul.suffix ]]'
+  
+    api_port: 2[[ crypto.SHA1 (printf "%s%s" .instance .consul.suffix) | regexp.Replace "[^\\d]" "" | regexp.Replace "^0*" "" | strings.Trunc 4 ]]
+  
+    api:
+      port: 2[[ crypto.SHA1 (printf "%s%s" .instance .consul.suffix) | regexp.Replace "[^\\d]" "" | regexp.Replace "^0*" "" | strings.Trunc 4 ]]
+      public_url: https://s3.example.org
+      traefik:
+        enabled: false
+        rule: 'Host(`[[ (urlParse .minio.server.api.public_url).Hostname ]]`) && !PathPrefix(`/minio/v2/metrics`)'
+        router: s3
+        csp: false
+        middlewares:
+          src-ip: ip-trusted@file
+  
+    console:
+      public_url: https://s3.example.org/admin
+      traefik:
+        enabled: false
+        router: console
+        strip_prefix: false
+        middlewares:
+          src-ip: ip-trusted@file
+  
+    metrics:
+      traefik:
+        enabled: false
+        rule: 'Host(`[[ (urlParse .minio.server.api.public_url).Hostname ]]`) && PathPrefix(`/minio/v2/metrics`)'
+        router: metrics
+        csp: false
+        middlewares:
+          src-ip: ip-metrics@file
+  
+    volumes:
+      data:
+        source: '[[ .instance ]]-data'
+        type: csi
+        per_alloc: true
+        destination: /data
+

vault/policies/minio.hcl

@@ -0,0 +1,7 @@
+path "[[ .vault.root ]]kv/data/service/[[ .instance ]]" {
+  capabilities = ["read"]
+}
+
+path "[[ .vault.pki.path ]]/issue/minio" {
+  capabilities = ["update"]
+}