monitoring/example/monitoring-exporters.nomad.hcl

job "monitoring-exporters" {

  datacenters = ["dc1"]
  region      = "global"


  # Run exporters. Use a separated job so exporters can run in a distinct node_pool
  group "exporters" {

    count = 1

    network {
      mode = "bridge"
      port "ping" {}
      port "blackbox" {}
      port "consul" {}
      port "cluster" {}
    }

    service {
      name = "ping-exporter"
      port = "ping"
      meta {
        alloc        = "${NOMAD_ALLOC_INDEX}"
        metrics-port = "${NOMAD_HOST_PORT_ping}"
      }
    }

    service {
      name = "blackbox-exporter"
      port = "blackbox"
      meta {
        alloc = "${NOMAD_ALLOC_INDEX}"
      }
    }

    service {
      name = "consul-exporter"
      port = "ping"
      meta {
        alloc        = "${NOMAD_ALLOC_INDEX}"
        metrics-port = "${NOMAD_HOST_PORT_consul}"
      }
    }

    service {
      name = "cluster-exporter"
      port = "cluster"
      meta {
        alloc = "${NOMAD_ALLOC_INDEX}"
      }
    }

    # Export consul services status to prometheus
    task "consul-exporter" {
      driver = "docker"

      config {
        image           = "danielberteaud/consul-exporter:0.11.0-2"
        readonly_rootfs = true
        pids_limit      = 30
        command         = "/local/consul-exporter"
      }



      # Use a template block instead of env {} so we can fetch values from vault
      template {
        data        = <<_EOT
LANG=fr_FR.utf8
TZ=Europe/Paris
_EOT
        destination = "secrets/.env"
        perms       = 400
        env         = true
      }


      vault {
        policies     = ["consul-exporter"]
        env          = false
        disable_file = true
        change_mode  = "noop"
      }


      template {
        data        = <<_EOT
#!/bin/sh

set -euo pipefail

exec consul_exporter \
  --web.listen-address=127.0.0.1:9107 \
  --consul.server=http://{{ sockaddr "GetInterfaceIP \"nomad\"" }}:8500 \
  --consul.request-limit=20

_EOT
        destination = "local/consul-exporter"
        perms       = 755
      }

      template {
        data        = <<_EOT
CONSUL_HTTP_TOKEN={{ with secret "consul/creds/consul-exporter" }}{{ .Data.token }}{{ end }}
_EOT
        destination = "secrets/.consul.env"
        uid         = 100000
        gid         = 100000
        perms       = 400
        env         = true
      }


      resources {
        cpu    = 20
        memory = 32
      }

    }

    # The cluster metrics exposes prometheus metrics from the various nodes of the cluster
    # Nomad, Consul and Vault
    # It also exposes the other exporters metrics with mTLS
    task "cluster-metrics-proxy" {
      driver = "docker"
      user   = 8685

      lifecycle {
        hook    = "poststart"
        sidecar = true
      }

      config {
        image           = "nginxinc/nginx-unprivileged:alpine"
        readonly_rootfs = true
        pids_limit      = 30
        # Mount the config in nginx conf dir
        volumes = [
          "secrets/metrics.conf:/etc/nginx/conf.d/default.conf"
        ]
        mount {
          type   = "tmpfs"
          target = "/tmp"
          tmpfs_options {
            size = 3000000
          }
        }

      }


      vault {
        policies     = ["cluster-exporter", "metrics"]
        env          = false
        disable_file = true
        change_mode  = "noop"
      }


      # This is the main nginx configuration, which will proxypass requests to the real metrics endpoints
      template {
        data          = <<_EOT

# Cluster exporter
server {
  listen {{ env "NOMAD_ALLOC_PORT_cluster" }} ssl;
  http2 on;

  ssl_certificate /secrets/metrics.bundle.pem;
  ssl_certificate_key /secrets/metrics.bundle.pem;
  ssl_client_certificate /local/monitoring.ca.pem;
  ssl_verify_client on;
  ssl_protocols TLSv1.2 TLSv1.3;
  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
  ssl_session_cache shared:SSL:10m;
  ssl_session_timeout 1h;
  ssl_session_tickets off;
  gzip on;
  gzip_types
    text/plain;
  gzip_vary on;

  server_tokens off;

  if ($request_method !~ ^(GET|HEAD)$ ) {
    return 405;
  }

  set $consul_token "{{ with secret "consul/creds/cluster-exporter" }}{{ .Data.token }}{{ end }}";

{{- range service "nomad-client" }}
  location /nomad-client/{{ .Node }} {
    proxy_pass https://{{ .Address }}:{{ .Port }}/v1/metrics?format=prometheus;
    proxy_ssl_certificate /secrets/nomad_client_bundle.pem;
    proxy_ssl_certificate_key /secrets/nomad_client_bundle.pem;
    proxy_ssl_verify on;
    proxy_ssl_name client.{{ env "NOMAD_REGION" }}.nomad;
    proxy_ssl_trusted_certificate /local/nomad_ca.crt;
  }
{{- end }}

{{- range service "nomad" }}
  {{- if .Tags | contains "http" }}
  location /nomad/{{ .Node }} {
    proxy_pass https://{{ .Address }}:{{ .Port }}/v1/metrics?format=prometheus;
    proxy_ssl_certificate /secrets/nomad_client_bundle.pem;
    proxy_ssl_certificate_key /secrets/nomad_client_bundle.pem;
    proxy_ssl_verify on;
    proxy_ssl_name server.{{ env "NOMAD_REGION" }}.nomad;
    proxy_ssl_trusted_certificate /local/nomad_ca.crt;
  }
  {{- end }}
{{- end }}

{{- range service "consul" }}
  location /consul/{{ .Node }} {
    proxy_pass https://{{ .Address }}:8501/v1/agent/metrics?format=prometheus;
    proxy_set_header X-Consul-Token $consul_token;
    proxy_ssl_certificate /secrets/consul_client_bundle.pem;
    proxy_ssl_certificate_key /secrets/consul_client_bundle.pem;
    proxy_ssl_verify off;
    proxy_ssl_trusted_certificate /local/consul_ca.crt;
  }
{{- end }}

{{- range service "vault" }}
  location /vault/{{ .Node }} {
    proxy_pass https://{{ .Address }}:{{ .Port }}/v1/sys/metrics?format=prometheus;
    proxy_ssl_verify on;
    proxy_ssl_trusted_certificate /etc/ssl/cert.pem;
    proxy_set_header X-Forwarded-For "$proxy_add_x_forwarded_for";
    proxy_set_header X-Real-IP "$remote_addr";
    proxy_set_header X-Forwarded-Proto "$scheme";
    proxy_set_header X-Scheme "$scheme";
    proxy_set_header X-Forwarded-Host "$host";
    proxy_set_header X-Forwarded-Port "$server_port";
  }
{{- end }}

  location / {
    root /usr/share/nginx/html;
    index index.html;
  }
}

# Ping exporter
server {
  listen {{ env "NOMAD_ALLOC_PORT_ping" }} ssl;
  http2 on;

  ssl_certificate /secrets/metrics.bundle.pem;
  ssl_certificate_key /secrets/metrics.bundle.pem;
  ssl_client_certificate /local/monitoring.ca.pem;
  ssl_verify_client on;
  ssl_protocols TLSv1.2 TLSv1.3;
  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
  ssl_session_cache shared:SSL:10m;
  ssl_session_timeout 1h;
  ssl_session_tickets off;
  gzip on;
  gzip_types
    text/plain;
  gzip_vary on;

  server_tokens off;

  if ($request_method !~ ^(GET|HEAD)$ ) {
    return 405;
  }
  location /metrics {
    proxy_pass http://127.0.0.1:9427;
  }
}

# Blackbox exporter
server {
  listen {{ env "NOMAD_ALLOC_PORT_blackbox" }} ssl;
  http2 on;

  ssl_certificate /secrets/metrics.bundle.pem;
  ssl_certificate_key /secrets/metrics.bundle.pem;
  ssl_client_certificate /local/monitoring.ca.pem;
  ssl_verify_client on;
  ssl_protocols TLSv1.2 TLSv1.3;
  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
  ssl_session_cache shared:SSL:10m;
  ssl_session_timeout 1h;
  ssl_session_tickets off;
  gzip on;
  gzip_types
    text/plain;
  gzip_vary on;

  server_tokens off;

  if ($request_method !~ ^(GET|HEAD)$ ) {
    return 405;
  }

  location / {
    proxy_pass http://127.0.0.1:9115;
  }
}

# Consul exporter
server {
  listen {{ env "NOMAD_ALLOC_PORT_consul" }} ssl;
  http2 on;

  ssl_certificate /secrets/metrics.bundle.pem;
  ssl_certificate_key /secrets/metrics.bundle.pem;
  ssl_client_certificate /local/monitoring.ca.pem;
  ssl_verify_client on;
  ssl_protocols TLSv1.2 TLSv1.3;
  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
  ssl_session_cache shared:SSL:10m;
  ssl_session_timeout 1h;
  ssl_session_tickets off;
  gzip on;
  gzip_types
    text/plain;
  gzip_vary on;

  server_tokens off;

  if ($request_method !~ ^(GET|HEAD)$ ) {
    return 405;
  }
  location /metrics {
    proxy_pass http://127.0.0.1:9107;
  }
}

_EOT
        destination   = "secrets/metrics.conf"
        perms         = "0440"
        uid           = 108685
        gid           = 100000
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }

      # Get certificate to add mTLS to metrics endpoints
      template {
        data          = <<_EOT
{{- with pkiCert "pki/monitoring/issue/metrics" (printf "ip_sans=%s" (env "NOMAD_HOST_IP_cluster")) }}
{{ .Cert }}
{{ .Key }}
{{- end }}
_EOT
        destination   = "secrets/metrics.bundle.pem"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }

      # Get the CA for the monitoring PKI
      template {
        data        = <<_EOT
{{ with secret "pki/monitoring/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
        destination = "local/monitoring.ca.pem"
      }

      # Get a Nomad client certificate
      template {
        data          = <<_EOT
{{- with pkiCert "pki/nomad/issue/cluster-exporter" "common_name=metrics-proxy.nomad.consul" "ttl=24h" }}
{{ .Data.Cert }}
{{ .Data.Key }}
{{- end }}
_EOT
        destination   = "secrets/nomad_client_bundle.pem"
        perms         = "0400"
        uid           = 108685
        gid           = 100000
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }

      # The CA chain for Nomad
      template {
        data        = <<_EOT
{{ with secret "pki/nomad/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
        destination = "local/nomad_ca.crt"
      }

      # Same for Consul
      template {
        data          = <<_EOT
{{- with pkiCert "pki/consul/issue/cluster-exporter" "common_name=metrics-proxy.consul.consul" "ttl=24h" }}
{{ .Data.Cert }}
{{ .Data.Key }}
{{- end }}
_EOT
        destination   = "secrets/consul_client_bundle.pem"
        perms         = "0400"
        uid           = 108685
        gid           = 100000
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
      template {
        data        = <<_EOT
{{ with secret "pki/consul/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
        destination = "local/consul_ca.crt"
      }


      resources {
        cpu        = 10
        memory     = 24
        memory_max = 32
      }

    }
  }
}