diff --git a/TODO.md b/TODO.md index d05f589..cfcc974 100644 --- a/TODO.md +++ b/TODO.md @@ -34,11 +34,12 @@ - vector-aggregator - vector-agent (dans job agent) - ~~loki (modulariser ou laisser en monolithique ?)~~ - - grafana + - ~~grafana~~ - ~~cluster-metrics (job exporters)~~ - questions / various - - prom rules: keep or move to a -conf bundle ? + - ~~prom rules: keep or move to a -conf bundle ?~~ - ~~config alertes am (recipient + routing)~~ - ~~http and tcp probes, as exporters are now in a dedicated job~~ - alertmanager & rules for loki + - bootstrap grafana diff --git a/example/TODO.md b/example/TODO.md index d05f589..cfcc974 100644 --- a/example/TODO.md +++ b/example/TODO.md @@ -34,11 +34,12 @@ - vector-aggregator - vector-agent (dans job agent) - ~~loki (modulariser ou laisser en monolithique ?)~~ - - grafana + - ~~grafana~~ - ~~cluster-metrics (job exporters)~~ - questions / various - - prom rules: keep or move to a -conf bundle ? + - ~~prom rules: keep or move to a -conf bundle ?~~ - ~~config alertes am (recipient + routing)~~ - ~~http and tcp probes, as exporters are now in a dedicated job~~ - alertmanager & rules for loki + - bootstrap grafana diff --git a/example/prep.d/10-montoring-rand-secrets b/example/prep.d/10-montoring-rand-secrets index c8e3b01..61d5b10 100755 --- a/example/prep.d/10-montoring-rand-secrets +++ b/example/prep.d/10-montoring-rand-secrets @@ -10,9 +10,10 @@ RAND_CMD="tr -dc A-Za-z0-9\-_\/=~\.+ < /dev/urandom | head -c 50" if ! vault kv list $(dirname ${VAULT_KV_PATH}) 2>/dev/null | grep -q -E "^$(basename ${VAULT_KV_PATH})\$"; then vault kv put ${VAULT_KV_PATH} \ secret_key="$(sh -c "${RAND_CMD}")" \ + initial_admin_pwd="$(sh -c "${RAND_CMD}")" \ fi -for SECRET in secret_key; do +for SECRET in secret_key initial_admin_pwd; do if ! vault kv get -field ${SECRET} ${VAULT_KV_PATH} >/dev/null 2>&1; then vault kv patch ${VAULT_KV_PATH} \ ${SECRET}=$(sh -c "${RAND_CMD}") diff --git a/example/services.nomad.hcl b/example/services.nomad.hcl index bea656b..69b99bf 100644 --- a/example/services.nomad.hcl +++ b/example/services.nomad.hcl @@ -1910,7 +1910,7 @@ _EOT } } - group "interface" { + group "grafana" { shutdown_delay = "6s" @@ -2141,6 +2141,7 @@ _EOT # Use a template block instead of env {} so we can fetch values from vault template { data = <<_EOT +GF_SECURITY_ADMIN_PASSWORD={{ with secret "kv/service/monitoring/grafana" }}{{ .Data.data.initial_admin_pwd }}{{ end }} LANG=fr_FR.utf8 TZ=Europe/Paris _EOT @@ -2185,6 +2186,8 @@ secret_key = {{ with secret "kv/service/monitoring/grafana" }}{{ .Data.data.secr [dataproxy] timeout = 120 +[feature_toggles] + _EOT destination = "secrets/grafana.ini" uid = 103000 diff --git a/services.nomad.hcl b/services.nomad.hcl index a17ba3a..7c00237 100644 --- a/services.nomad.hcl +++ b/services.nomad.hcl @@ -559,7 +559,7 @@ _EOT } } - group "interface" { + group "grafana" { [[- $c := merge .monitoring.grafana .monitoring . ]] shutdown_delay = "6s" diff --git a/templates/grafana/grafana.ini b/templates/grafana/grafana.ini index 88941dc..b7dda88 100644 --- a/templates/grafana/grafana.ini +++ b/templates/grafana/grafana.ini @@ -35,3 +35,8 @@ secret_key = {{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]/grafana [dataproxy] timeout = 120 + +[feature_toggles] +[[- range $feature, $value := .feature_toggles ]] +[[ $feature ]] = [[ $value ]] +[[- end ]] diff --git a/templates/prometheus/prometheus.yml b/templates/prometheus/prometheus.yml index 7b69591..aa58edc 100644 --- a/templates/prometheus/prometheus.yml +++ b/templates/prometheus/prometheus.yml @@ -52,7 +52,7 @@ scrape_configs: params: module: ["http_2xx"] static_configs: - {{ range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }} + {{- range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }} - targets: [[- range $http_probe := .exporters.blackbox.http_probes ]] - [[ $http_probe ]] @@ -63,8 +63,8 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: {{ .Address }}:{{ .Port }} - {{ end }} + replacement: {{ $instance.Address }}:{{ $instance.Port }} + {{- end }} {{- end }} [[- end ]] @@ -81,9 +81,10 @@ scrape_configs: params: module: ["tcp_connect"] static_configs: - { range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }} + {{- range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }} + - targets: [[- range $target := .exporters.blackbox.tcp_probes ]] - - [[ $target ]] + - [[ $target ]] [[- end ]] relabel_configs: - source_labels: [__address__] @@ -91,11 +92,25 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: {{ .Address }}:{{ .Port }} - {{ end }} + replacement: {{ $instance.Address }}:{{ $instance.Port }} + {{- end }} {{- end }} [[- end ]] +[[- if gt (len .exporters.ping.probes) 0 ]] + # Ping exporter + - job_name: ping + scheme: https + tls_config: + ca_file: /local/monitoring.ca.pem + cert_file: /secrets/prometheus.bundle.pem + key_file: /secrets/prometheus.bundle.pem + static_configs: + {{- range $idx, $instance := service "ping-exporter[[ .consul.suffix ]]" }} + - targets: ["{{ $instance.Address }}:{{ $instance.Port }}"] + {{- end }} +[[- end ]] + # Cluster services - job_name: cluster-services scheme: https diff --git a/variables.yml b/variables.yml index 9075974..afe2c06 100644 --- a/variables.yml +++ b/variables.yml @@ -10,6 +10,7 @@ vault: - path: grafana fields: - secret_key + - initial_admin_pwd monitoring: @@ -188,7 +189,8 @@ monitoring: grafana: version: 10.4.1 image: '[[ .docker.repo ]]grafana:[[ .monitoring.grafana.version ]]-1' - env: {} + env: + GF_SECURITY_ADMIN_PASSWORD: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]/grafana" }}{{ .Data.data.initial_admin_pwd }}{{ end }}' resources: cpu: 100 memory: 256 @@ -198,6 +200,7 @@ monitoring: #- ddurieux-glpi-app - grafana-clock-panel - grafana-piechart-panel + feature_toggles: {} traefik: enabled: true router: grafana