From ad1b762e44bce91825b1fecebdfeb6b07d18821d Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Wed, 27 Mar 2024 23:09:39 +0100 Subject: [PATCH] [squid] Add support for prometheus exporter --- example/images/squid-exporter/Dockerfile | 25 +++++++++++ example/prep.d/10-squid-rand-secrets | 22 ++++++++++ example/squid.nomad.hcl | 54 ++++++++++++++++++++++++ example/vault/policies/squid.hcl | 3 ++ images/squid-exporter/Dockerfile | 25 +++++++++++ prep.d/10-squid-rand-secrets | 5 +++ squid.nomad.hcl | 53 +++++++++++++++++++++++ templates/auth.tpl | 1 + templates/squid.conf | 22 ++++++++++ variables.yml | 21 +++++++++ vault/policies/squid.hcl | 3 ++ 11 files changed, 234 insertions(+) create mode 100644 example/images/squid-exporter/Dockerfile create mode 100755 example/prep.d/10-squid-rand-secrets create mode 100644 example/vault/policies/squid.hcl create mode 100644 images/squid-exporter/Dockerfile create mode 100755 prep.d/10-squid-rand-secrets create mode 100644 templates/squid.conf create mode 100644 vault/policies/squid.hcl diff --git a/example/images/squid-exporter/Dockerfile b/example/images/squid-exporter/Dockerfile new file mode 100644 index 0000000..1797f7e --- /dev/null +++ b/example/images/squid-exporter/Dockerfile @@ -0,0 +1,25 @@ +FROM danielberteaud/alpine:24.3-1 +MAINTAINER Daniel Berteaud + +ARG EXPORTER_VERSION=1.11.0 + +ADD --chmod=755 --chown=root:root https://github.com/boynux/squid-exporter/releases/download/v${EXPORTER_VERSION}/squid-exporter-linux-amd64 /usr/local/bin/squid-exporter + +ENV SQUID_EXPORTER_LISTEN=0.0.0.0:9301 \ + SQUID_EXPORTER_METRICS_PATH=/metrics \ + SQUID_HOSTNAME=127.0.0.1 \ + SQUID_PORT=3128 + +RUN set -euxo pipefail &&\ + addgroup --gid 9301 squid-exporter &&\ + adduser --system \ + --ingroup squid-exporter \ + --disabled-password \ + --uid 9301 \ + --home /home/squid-exporter \ + --shell /sbin/nologin \ + squid-exporter + +USER squid-exporter +EXPOSE 9301 +CMD ["squid-exporter"] diff --git a/example/prep.d/10-squid-rand-secrets b/example/prep.d/10-squid-rand-secrets new file mode 100755 index 0000000..fc1b7bf --- /dev/null +++ b/example/prep.d/10-squid-rand-secrets @@ -0,0 +1,22 @@ +#!/bin/sh + +set -euo pipefail + +# vim: syntax=sh + +export LC_ALL=C +VAULT_KV_PATH=kv/service/squid +RAND_CMD="tr -dc A-Za-z0-9\-_\/=~\.+ < /dev/urandom | head -c 50" +if ! vault kv list $(dirname ${VAULT_KV_PATH}) 2>/dev/null | grep -q -E "^$(basename ${VAULT_KV_PATH})\$"; then + vault kv put ${VAULT_KV_PATH} \ + manager_pwd="$(sh -c "${RAND_CMD}")" \ + +fi +for SECRET in manager_pwd; do + if ! vault kv get -field ${SECRET} ${VAULT_KV_PATH} >/dev/null 2>&1; then + vault kv patch ${VAULT_KV_PATH} \ + ${SECRET}=$(sh -c "${RAND_CMD}") + fi +done + + diff --git a/example/squid.nomad.hcl b/example/squid.nomad.hcl index 07208af..5b8744f 100644 --- a/example/squid.nomad.hcl +++ b/example/squid.nomad.hcl @@ -5,6 +5,8 @@ job "squid" { region = "global" + + group "squid" { network { mode = "bridge" @@ -15,6 +17,16 @@ job "squid" { service { name = "squid" port = 3128 + meta { + alloc = "${NOMAD_ALLOC_INDEX}" + datacenter = "${NOMAD_DC}" + group = "${NOMAD_GROUP_NAME}" + job = "${NOMAD_JOB_NAME}" + namespace = "${NOMAD_NAMESPACE}" + node = "${node.unique.name}" + region = "${NOMAD_REGION}" + } + connect { sidecar_service { disable_default_tcp_check = true @@ -50,11 +62,23 @@ job "squid" { readonly_rootfs = true pids_limit = 100 volumes = [ + "local/squid.conf:/etc/squid/squid.conf:ro", "secrets/:/etc/squid/conf.d", "local/filter-acl.sh:/entrypoint.d/30-filter-acl.sh:ro" ] } + + + + vault { + policies = ["squid"] + env = false + disable_file = true + change_mode = "noop" + } + + env { SQUID_LISTS_DIR = "/local/lists" SQUID_CONF_5_auth_param = "basic program /usr/lib/squid/basic_ncsa_auth /secrets/auth" @@ -85,6 +109,35 @@ _EOT perms = 755 } + template { + data = <<_EOT +max_filedescriptors 8192 +pid_filename none +http_port 3128 + +# Log on stdout +access_log stdio:/dev/stdout combined + +# NCSA auth +auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/auth +auth_param basic children 2 startup=2 idle=1 +auth_param basic credentialsttl 1 hours + +# Allow squid manager +acl auth_squid_manager proxy_auth squid_manager +http_access allow manager localhost auth_squid_manager +# Deny cache manager to anyone else +http_access deny manager + +# Include config fragment +include /etc/squid/conf.d/*.conf + + + +_EOT + destination = "local/squid.conf" + } + template { data = <<_EOT #!/bin/sh @@ -110,6 +163,7 @@ _EOT template { data = <<_EOT +squid_manager:{{ with secret "kv/service/squid" }}{{ sprig_bcrypt .Data.data.manager_pwd }}{{ end }} {{- range services }} {{- if not (.Name | regexMatch ".*sidecar\\-proxy$") }} {{ .Name }}:{{ sprig_bcrypt .Name }} diff --git a/example/vault/policies/squid.hcl b/example/vault/policies/squid.hcl new file mode 100644 index 0000000..d82aba8 --- /dev/null +++ b/example/vault/policies/squid.hcl @@ -0,0 +1,3 @@ +path "kv/data/service/squid" { + capabilities = ["read"] +} diff --git a/images/squid-exporter/Dockerfile b/images/squid-exporter/Dockerfile new file mode 100644 index 0000000..5d1209a --- /dev/null +++ b/images/squid-exporter/Dockerfile @@ -0,0 +1,25 @@ +FROM [[ .docker.repo ]][[ .docker.base_images.alpine.image ]] +MAINTAINER [[ .docker.maintainer ]] + +ARG EXPORTER_VERSION=[[ .squid.exporter.version ]] + +ADD --chmod=755 --chown=root:root https://github.com/boynux/squid-exporter/releases/download/v${EXPORTER_VERSION}/squid-exporter-linux-amd64 /usr/local/bin/squid-exporter + +ENV SQUID_EXPORTER_LISTEN=0.0.0.0:9301 \ + SQUID_EXPORTER_METRICS_PATH=/metrics \ + SQUID_HOSTNAME=127.0.0.1 \ + SQUID_PORT=3128 + +RUN set -euxo pipefail &&\ + addgroup --gid 9301 squid-exporter &&\ + adduser --system \ + --ingroup squid-exporter \ + --disabled-password \ + --uid 9301 \ + --home /home/squid-exporter \ + --shell /sbin/nologin \ + squid-exporter + +USER squid-exporter +EXPOSE 9301 +CMD ["squid-exporter"] diff --git a/prep.d/10-squid-rand-secrets b/prep.d/10-squid-rand-secrets new file mode 100755 index 0000000..2a5ef21 --- /dev/null +++ b/prep.d/10-squid-rand-secrets @@ -0,0 +1,5 @@ +#!/bin/sh + +set -euo pipefail + +[[ template "common/vault.rand_secrets" merge .squid . ]] diff --git a/squid.nomad.hcl b/squid.nomad.hcl index 0b02c7a..34024e7 100644 --- a/squid.nomad.hcl +++ b/squid.nomad.hcl @@ -7,6 +7,9 @@ job "[[ .instance ]]" { group "squid" { network { mode = "bridge" +[[- if conv.ToBool $c.prometheus.enabled ]] + port "metrics" {} +[[- end ]] } count = [[ $c.count ]] @@ -14,6 +17,7 @@ job "[[ .instance ]]" { service { name = "[[ .instance ]][[ .consul.suffix ]]" port = 3128 +[[ template "common/service_meta" $c ]] [[ template "common/connect" $c ]] } @@ -25,11 +29,15 @@ job "[[ .instance ]]" { readonly_rootfs = true pids_limit = 100 volumes = [ + "local/squid.conf:/etc/squid/squid.conf:ro", "secrets/:/etc/squid/conf.d", "local/filter-acl.sh:/entrypoint.d/30-filter-acl.sh:ro" ] } +[[ template "common/artifacts" $c ]] +[[ template "common/vault.policies" $c ]] + env { SQUID_LISTS_DIR = "/local/lists" SQUID_CONF_5_auth_param = "basic program /usr/lib/squid/basic_ncsa_auth /secrets/auth" @@ -46,6 +54,13 @@ _EOT perms = 755 } + template { + data =<<_EOT +[[ template "squid/squid.conf" $c ]] +_EOT + destination = "local/squid.conf" + } + template { data =<<_EOT [[ template "squid/reload.sh.tpl" $c ]] @@ -107,5 +122,43 @@ _EOT [[ template "common/file_env" $c ]] [[ template "common/resources" $c ]] } + +[[- if conv.ToBool $c.prometheus.enabled ]] + +[[ template "common/task.metrics_proxy" $c ]] + +[[- $c := merge $c.exporter $c ]] + + task "exporter" { + driver = "[[ $c.nomad.driver ]]" + + lifecycle { + hook = "poststart" + sidecar = true + } + + config { + image = "[[ $c.image ]]" + readonly_rootfs = true + pids_limit = 20 + } + +[[ template "common/vault.policies" $c ]] + + template { + data = <<_EOT +SQUID_EXPORTER_LISTEN=127.0.0.1:9301 +SQUID_LOGIN=squid_manager +SQUID_PASSWORD='{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ .Data.data.manager_pwd }}{{ end }}' +_EOT + destination = "secrets/.squid-exporter.env" + perms = 400 + env = true + } + +[[ template "common/resources" $c ]] + } + +[[- end ]] } } diff --git a/templates/auth.tpl b/templates/auth.tpl index 293417e..840f9ad 100644 --- a/templates/auth.tpl +++ b/templates/auth.tpl @@ -1,3 +1,4 @@ +squid_manager:{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]" }}{{ sprig_bcrypt .Data.data.manager_pwd }}{{ end }} {{- range services }} {{- if not (.Name | regexMatch ".*sidecar\\-proxy$") }} {{ .Name }}:{{ sprig_bcrypt .Name }} diff --git a/templates/squid.conf b/templates/squid.conf new file mode 100644 index 0000000..b86399a --- /dev/null +++ b/templates/squid.conf @@ -0,0 +1,22 @@ +max_filedescriptors 8192 +pid_filename none +http_port 3128 + +# Log on stdout +access_log stdio:/dev/stdout combined + +# NCSA auth +auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/auth +auth_param basic children 2 startup=2 idle=1 +auth_param basic credentialsttl 1 hours + +# Allow squid manager +acl auth_squid_manager proxy_auth squid_manager +http_access allow manager localhost auth_squid_manager +# Deny cache manager to anyone else +http_access deny manager + +# Include config fragment +include /etc/squid/conf.d/*.conf + + diff --git a/variables.yml b/variables.yml index a84eb69..5541c78 100644 --- a/variables.yml +++ b/variables.yml @@ -16,6 +16,13 @@ squid: cpu: 100 memory: 256 + vault: + policies: + - '[[ .instance ]][[ .consul.suffix ]]' + rand_secrets: + - fields: + - manager_pwd + # Env variables passed to the container # squid configuration can be passed with # @@ -97,3 +104,17 @@ squid: consul: connect: disable_default_tcp_check: true + + prometheus: + enabled: '[[ .prometheus.available ]]' + metrics_url: http://127.0.0.1:9301/metrics + + exporter: + version: 1.11.0 + image: '[[ .docker.repo ]]squid-exporter:[[ .squid.exporter.version ]]-1' + resources: + cpu: 10 + memory: 20 + vault: + policies: + - '[[ .instance ]][[ .consul.suffix ]]' diff --git a/vault/policies/squid.hcl b/vault/policies/squid.hcl new file mode 100644 index 0000000..6320284 --- /dev/null +++ b/vault/policies/squid.hcl @@ -0,0 +1,3 @@ +path "[[ .vault.root ]]kv/data/service/[[ .instance ]]" { + capabilities = ["read"] +}