nomad/minio
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use new nginx base image

Browse Source
This commit is contained in:
Daniel Berteaud
2025-04-27 15:35:32 +02:00
parent 3668cb34aa
commit 2ab8dda12d
4 changed files with 195 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

283
example/minio.nomad.hcl
View File

@@ -96,12 +96,16 @@ job "minio" {
}
config {
image = "nginxinc/nginx-unprivileged:alpine"
force_pull = true
volumes = ["local/nginx.conf:/etc/nginx/nginx.conf:ro"]
image = "danielberteaud/nginx:25.4-4"
volumes = ["local/nginx.conf:/etc/nginx/conf.d/default.conf:ro"]
pids_limit = 100
}
env {
TMPDIR = "/local/tmp"
NGINX_WORKERS = 2
}
vault {
role = "metrics"
disable_file = true
@@ -131,60 +135,42 @@ _EOT
template {
data = <<_EOT
worker_processes auto;
error_log /dev/stderr warn;
pid /secrets/nginx.pid;
events {
worker_connections 1024;
}
http {
proxy_temp_path /local/proxy_temp;
client_body_temp_path /local/client_temp;
fastcgi_temp_path /local/fastcgi_temp;
uwsgi_temp_path /local/uwsgi_temp;
scgi_temp_path /local/scgi_temp;
include /etc/nginx/mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /dev/stdout main;
server {
listen {{ env "NOMAD_ALLOC_PORT_metrics" }} ssl;
http2 on;
ssl_certificate /secrets/metrics.bundle.pem;
ssl_certificate_key /secrets/metrics.bundle.pem;
ssl_client_certificate /local/monitoring.ca.pem;
ssl_verify_client on;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1h;
ssl_session_tickets off;
gzip on;
gzip_types
text/plain;
gzip_vary on;
server_tokens off;
if ($request_method !~ ^(GET|HEAD)$ ) {
return 405;
}
location /envoy {
proxy_pass http://127.0.0.1:7462/metrics;
proxy_http_version 1.1;
}
location /bucket {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/bucket;
proxy_http_version 1.1;
}
location /cluster {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/cluster;
proxy_http_version 1.1;
}
location /resource {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/resource;
proxy_http_version 1.1;
}
}
server {
listen {{ env "NOMAD_ALLOC_PORT_metrics" }} ssl;
http2 on;
ssl_certificate /secrets/metrics.bundle.pem;
ssl_certificate_key /secrets/metrics.bundle.pem;
ssl_client_certificate /local/monitoring.ca.pem;
ssl_verify_client on;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1h;
ssl_session_tickets off;
gzip on;
gzip_types
text/plain;
gzip_vary on;
server_tokens off;
if ($request_method !~ ^(GET|HEAD)$ ) {
return 405;
}
location /envoy {
proxy_pass http://127.0.0.1:7462/metrics;
proxy_http_version 1.1;
}
location /bucket {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/bucket;
proxy_http_version 1.1;
}
location /cluster {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/cluster;
proxy_http_version 1.1;
}
location /resource {
proxy_pass http://127.0.0.1:8000/minio/v2/metrics/resource;
proxy_http_version 1.1;
}
}
_EOT
destination = "local/nginx.conf"
@@ -378,11 +364,15 @@ _EOT
}
config {
image = "nginxinc/nginx-unprivileged:alpine"
force_pull = true
volumes = ["local/nginx.conf:/etc/nginx/nginx.conf:ro"]
image = "danielberteaud/nginx:25.4-4"
volumes = ["local/nginx.conf:/etc/nginx/conf.d/default.conf:ro"]
pids_limit = 100
}
env {
TMPDIR = "/local/tmp"
NGINX_WORKERS = 2
}
lifecycle {
hook = "poststart"
sidecar = true
@@ -417,48 +407,30 @@ _EOT
template {
data = <<_EOT
worker_processes auto;
error_log /dev/stderr warn;
pid /secrets/nginx.pid;
events {
worker_connections 1024;
}
http {
proxy_temp_path /local/proxy_temp;
client_body_temp_path /local/client_temp;
fastcgi_temp_path /local/fastcgi_temp;
uwsgi_temp_path /local/uwsgi_temp;
scgi_temp_path /local/scgi_temp;
include /etc/nginx/mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /dev/stdout main;
server {
listen {{ env "NOMAD_ALLOC_PORT_metrics" }} ssl;
http2 on;
ssl_certificate /secrets/metrics.bundle.pem;
ssl_certificate_key /secrets/metrics.bundle.pem;
ssl_client_certificate /local/monitoring.ca.pem;
ssl_verify_client on;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1h;
ssl_session_tickets off;
gzip on;
gzip_types
text/plain;
gzip_vary on;
server_tokens off;
if ($request_method !~ ^(GET|HEAD)$ ) {
return 405;
}
location /envoy {
proxy_pass http://127.0.0.1:7462/metrics;
proxy_http_version 1.1;
}
}
server {
listen {{ env "NOMAD_ALLOC_PORT_metrics" }} ssl;
http2 on;
ssl_certificate /secrets/metrics.bundle.pem;
ssl_certificate_key /secrets/metrics.bundle.pem;
ssl_client_certificate /local/monitoring.ca.pem;
ssl_verify_client on;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_session_cache shared:SSL:10m;
ssl_session_timeout 1h;
ssl_session_tickets off;
gzip on;
gzip_types
text/plain;
gzip_vary on;
server_tokens off;
if ($request_method !~ ^(GET|HEAD)$ ) {
return 405;
}
location /envoy {
proxy_pass http://127.0.0.1:7462/metrics;
proxy_http_version 1.1;
}
}
_EOT
destination = "local/nginx.conf"
@@ -477,7 +449,6 @@ _EOT
# We could instead use two connect services (so 2 envoy sidecars), but nginx is lighter
# It's also used to expose the service as plain http for the service mesh
task "nginx" {
driver = "docker"
lifecycle {
@@ -486,8 +457,7 @@ _EOT
}
config {
image = "nginxinc/nginx-unprivileged:alpine"
force_pull = true
image = "danielberteaud/nginx:25.4-4"
readonly_rootfs = true
pids_limit = 100
@@ -499,75 +469,62 @@ _EOT
}
}
volumes = ["local/nginx.conf:/etc/nginx/nginx.conf:ro"]
volumes = ["local/nginx.conf:/etc/nginx/conf.d/default.conf:ro"]
}
env {
TMPDIR = "/local/tmp"
}
template {
data = <<_EOT
worker_processes auto;
error_log /dev/stderr warn;
pid /secrets/nginx.pid;
events {
worker_connections 1024;
}
http {
proxy_temp_path /local/proxy_temp;
client_body_temp_path /local/client_temp;
fastcgi_temp_path /local/fastcgi_temp;
uwsgi_temp_path /local/uwsgi_temp;
scgi_temp_path /local/scgi_temp;
include /etc/nginx/mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /dev/stdout main;
# Proxy for both S3 API and Consol
server {
listen 127.0.0.1:8000 default;
server_name _;
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
server {
listen 127.0.0.1:8000 default;
server_name _;
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
location /admin/ws/ {
rewrite ^/admin/ws/(.*) /ws/$1 break;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_pass http://127.0.0.1:9001;
}
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_pass http://127.0.0.1:9001;
}
location /admin/ {
rewrite ^/admin/(.*) /$1 break;
proxy_pass http://127.0.0.1:9001;
}
location /admin/ {
rewrite ^/admin/(.*) /$1 break;
proxy_pass http://127.0.0.1:9001;
}
# Proxy the S3 API as it shares the same vhost as the console
location / {
proxy_pass http://127.0.0.1:9000;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
# Proxy the S3 API as it shares the same vhost as the console
location / {
proxy_pass http://127.0.0.1:9000;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
}

7
minio.nomad.hcl
View File

@@ -92,7 +92,6 @@ job "[[ .instance ]]" {
# It's also used to expose the service as plain http for the service mesh
task "nginx" {
[[- $d := merge $c.nginx $c ]]
driver = "[[ $d.nomad.driver ]]"
lifecycle {
@@ -104,7 +103,11 @@ job "[[ .instance ]]" {
[[ template "common/image" $d ]]
pids_limit = 100
[[ template "common/tmpfs" "/tmp" ]]
volumes = ["local/nginx.conf:/etc/nginx/nginx.conf:ro"]
volumes = ["local/nginx.conf:/etc/nginx/conf.d/default.conf:ro"]
}
env {
TMPDIR = "/local/tmp"
}
template {

157
templates/nginx.conf
View File

@@ -1,21 +1,3 @@
worker_processes auto;
error_log /dev/stderr warn;
pid /secrets/nginx.pid;
events {
worker_connections 1024;
}
http {
proxy_temp_path /local/proxy_temp;
client_body_temp_path /local/client_temp;
fastcgi_temp_path /local/fastcgi_temp;
uwsgi_temp_path /local/uwsgi_temp;
scgi_temp_path /local/scgi_temp;
include /etc/nginx/mime.types;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /dev/stdout main;
[[- $proto := "http" ]]
[[- $port := "9000" ]]
[[- if gt .count 1 ]]
@@ -25,64 +7,64 @@ http {
[[- if ne ((urlParse .console.public_url).Hostname) ((urlParse .api.public_url).Hostname) ]]
# S3 API proxy
server {
listen 127.0.0.1:8000 default;
server_name _;
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
location / {
proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
}
# S3 API proxy
server {
listen 127.0.0.1:8000 default;
server_name _;
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
location / {
proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
}
}
# Console proxy
[[- else ]]
# Proxy for both S3 API and Consol
[[- end ]]
server {
server {
[[- if eq ((urlParse .console.public_url).Hostname) ((urlParse .api.public_url).Hostname) ]]
listen 127.0.0.1:8000 default;
server_name _;
listen 127.0.0.1:8000 default;
server_name _;
[[- else ]]
listen 127.0.0.1:8000;
server_name [[ (urlParse .console.public_url).Hostname ]];
listen 127.0.0.1:8000;
server_name [[ (urlParse .console.public_url).Hostname ]];
[[- end ]]
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
server_tokens off;
root /usr/share/html;
set_real_ip_from 127.0.0.1;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
ignore_invalid_headers off;
client_max_body_size 0;
proxy_buffering off;
proxy_request_buffering off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
chunked_transfer_encoding off;
[[- if eq (.console.public_url | regexp.Replace "/$" "") (.api.public_url | regexp.Replace "/$" "") ]]
[[- fail "Console and API must use a different host, or path" ]]
@@ -103,32 +85,31 @@ http {
rewrite ^[[ $console ]]ws/(.*) /ws/$1 break;
[[- end ]]
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_pass [[ $proto ]]://127.0.0.1:9001;
}
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
proxy_http_version 1.1;
proxy_pass [[ $proto ]]://127.0.0.1:9001;
}
location [[ $console ]] {
location [[ $console ]] {
[[- if ne $console "/" ]]
rewrite ^[[ $console ]](.*) /$1 break;
rewrite ^[[ $console ]](.*) /$1 break;
[[- end ]]
proxy_pass [[ $proto ]]://127.0.0.1:9001;
}
proxy_pass [[ $proto ]]://127.0.0.1:9001;
}
[[- if eq ((urlParse .console.public_url).Hostname) ((urlParse .api.public_url).Hostname) ]]
# Proxy the S3 API as it shares the same vhost as the console
location / {
proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
[[- end ]]
# Proxy the S3 API as it shares the same vhost as the console
location / {
proxy_pass [[ $proto ]]://127.0.0.1:[[ $port ]];
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
[[- end ]]
}

2
variables.yml
View File

@@ -106,7 +106,7 @@ minio:
# A small nginx proxy is used to multiplexe the S3 API and the Console on the same port
nginx:
image: nginxinc/nginx-unprivileged:alpine
image: '[[ .docker.repo ]][[ .docker.base_images.nginx.image ]]'
resources:
cpu: 20
memory: 32