Small fixes

TODO.md

@@ -34,11 +34,12 @@
   - vector-aggregator
   - vector-agent (dans job agent)
   - ~~loki (modulariser ou laisser en monolithique ?)~~
-  - grafana
+  - ~~grafana~~
   - ~~cluster-metrics (job exporters)~~
 
 - questions / various
-  - prom rules: keep or move to a -conf bundle ?
+  - ~~prom rules: keep or move to a -conf bundle ?~~
   - ~~config alertes am (recipient + routing)~~
   - ~~http and tcp probes, as exporters are now in a dedicated job~~
   - alertmanager & rules for loki
+  - bootstrap grafana

example/TODO.md

@@ -34,11 +34,12 @@
   - vector-aggregator
   - vector-agent (dans job agent)
   - ~~loki (modulariser ou laisser en monolithique ?)~~
-  - grafana
+  - ~~grafana~~
   - ~~cluster-metrics (job exporters)~~
 
 - questions / various
-  - prom rules: keep or move to a -conf bundle ?
+  - ~~prom rules: keep or move to a -conf bundle ?~~
   - ~~config alertes am (recipient + routing)~~
   - ~~http and tcp probes, as exporters are now in a dedicated job~~
   - alertmanager & rules for loki
+  - bootstrap grafana

example/prep.d/10-montoring-rand-secrets

@@ -10,9 +10,10 @@ RAND_CMD="tr -dc A-Za-z0-9\-_\/=~\.+ < /dev/urandom | head -c 50"
 if ! vault kv list $(dirname ${VAULT_KV_PATH}) 2>/dev/null | grep -q -E "^$(basename ${VAULT_KV_PATH})\$"; then
   vault kv put ${VAULT_KV_PATH} \
     secret_key="$(sh -c "${RAND_CMD}")" \
+    initial_admin_pwd="$(sh -c "${RAND_CMD}")" \
 
 fi
-for SECRET in secret_key; do
+for SECRET in secret_key initial_admin_pwd; do
   if ! vault kv get -field ${SECRET} ${VAULT_KV_PATH} >/dev/null 2>&1; then
     vault kv patch ${VAULT_KV_PATH} \
       ${SECRET}=$(sh -c "${RAND_CMD}")

example/services.nomad.hcl

@@ -1910,7 +1910,7 @@ _EOT
     }
   }
 
-  group "interface" {
+  group "grafana" {
 
     shutdown_delay = "6s"
 
@@ -2141,6 +2141,7 @@ _EOT
       # Use a template block instead of env {} so we can fetch values from vault
       template {
         data        = <<_EOT
+GF_SECURITY_ADMIN_PASSWORD={{ with secret "kv/service/monitoring/grafana" }}{{ .Data.data.initial_admin_pwd }}{{ end }}
 LANG=fr_FR.utf8
 TZ=Europe/Paris
 _EOT
@@ -2185,6 +2186,8 @@ secret_key = {{ with secret "kv/service/monitoring/grafana" }}{{ .Data.data.secr
 [dataproxy]
 timeout = 120
 
+[feature_toggles]
+
 _EOT
         destination = "secrets/grafana.ini"
         uid         = 103000

services.nomad.hcl

@@ -559,7 +559,7 @@ _EOT
     }
   }
 
-  group "interface" {
+  group "grafana" {
 [[- $c := merge .monitoring.grafana .monitoring . ]]
 
     shutdown_delay = "6s"

templates/grafana/grafana.ini

@@ -35,3 +35,8 @@ secret_key = {{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]/grafana
 
 [dataproxy]
 timeout = 120
+
+[feature_toggles]
+[[- range $feature, $value := .feature_toggles ]]
+[[ $feature ]] = [[ $value ]]
+[[- end ]]

templates/prometheus/prometheus.yml

@@ -52,7 +52,7 @@ scrape_configs:
     params:
       module: ["http_2xx"]
     static_configs:
-  {{ range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }}
+  {{- range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }}
       - targets:
   [[- range $http_probe := .exporters.blackbox.http_probes ]]
           - [[ $http_probe ]]
@@ -63,8 +63,8 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: instance
       - target_label: __address__
-        replacement: {{ .Address }}:{{ .Port }}
-  {{ end }}
+        replacement: {{ $instance.Address }}:{{ $instance.Port }}
+  {{- end }}
 {{- end }}
 [[- end ]]
 
@@ -81,9 +81,10 @@ scrape_configs:
     params:
       module: ["tcp_connect"]
     static_configs:
-  { range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }}
+  {{- range $idx, $instance := service "blackbox-exporter[[ .consul.suffix ]]" }}
+      - targets:
   [[- range $target := .exporters.blackbox.tcp_probes ]]
-      - [[ $target ]]
+         - [[ $target ]]
   [[- end ]]
     relabel_configs:
       - source_labels: [__address__]
@@ -91,11 +92,25 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: instance
       - target_label: __address__
-        replacement: {{ .Address }}:{{ .Port }}
-  {{ end }}
+        replacement: {{ $instance.Address }}:{{ $instance.Port }}
+  {{- end }}
 {{- end }}
 [[- end ]]
 
+[[- if gt (len .exporters.ping.probes) 0 ]]
+  # Ping exporter
+  - job_name: ping
+    scheme: https
+    tls_config:
+      ca_file: /local/monitoring.ca.pem
+      cert_file: /secrets/prometheus.bundle.pem
+      key_file: /secrets/prometheus.bundle.pem
+    static_configs:
+  {{- range $idx, $instance := service "ping-exporter[[ .consul.suffix ]]" }}
+      - targets: ["{{ $instance.Address }}:{{ $instance.Port }}"]
+  {{- end }}
+[[- end ]]
+
   # Cluster services
   - job_name: cluster-services
     scheme: https

variables.yml

@@ -10,6 +10,7 @@ vault:
     - path: grafana
       fields:
         - secret_key
+        - initial_admin_pwd
 
 monitoring:
 
@@ -188,7 +189,8 @@ monitoring:
   grafana:
     version: 10.4.1
     image: '[[ .docker.repo ]]grafana:[[ .monitoring.grafana.version ]]-1'
-    env: {}
+    env:
+      GF_SECURITY_ADMIN_PASSWORD: '{{ with secret "[[ .vault.root ]]kv/service/[[ .instance ]]/grafana" }}{{ .Data.data.initial_admin_pwd }}{{ end }}'
     resources:
       cpu: 100
       memory: 256
@@ -198,6 +200,7 @@ monitoring:
       #- ddurieux-glpi-app
       - grafana-clock-panel
       - grafana-piechart-panel
+    feature_toggles: {}
     traefik:
       enabled: true
       router: grafana