monitoring/agent.nomad.hcl

job "[[ .instance ]]-agent" {

[[- $c := merge .monitoring.agent .monitoring . ]]
[[ template "common/job_start" $c ]]
  type = "system"

  group "logs-collector" {

[[ $c := merge $c.vector $c ]]

    network {
      mode = "bridge"
      port "metrics" {}
    }

    # Try harder to restart tasks if they fail
    restart {
      attempts = 20
      interval = "5m"
      mode     = "delay"
    }

[[ template "common/volumes" $c ]]

    service {
      name = "vector-agent[[ .consul.suffix ]]"
[[ template "common/service_meta" $c ]]
[[ template "common/connect" $c ]]
    }

    task "nomad-vector-logger" {
[[- $n := merge $c.nomad_vector_logger $c ]]

      driver = "[[ $n.nomad.driver ]]"
      # Use a random user instead of root
      user = 3987

      config {
        image = "[[ $n.image ]]"
        readonly_rootfs = true
        pids_limit = 50
        # Nomad Vector Logger needs to run on the host's network namespace
        # so it can reach the Nomad Agent API on localhost:4646
        network_mode = "host"
        # Host network namespace requires disabling user namespace
        userns_mode = "host"
        command = "nomad-vector-logger"
        args = [
          "--config",
          "/local/nomad-vector-logger.toml"
        ]
      }

      # We want to run Nomad Vector Logger before vector agent
      lifecycle {
        hook    = "prestart"
        sidecar = true
      }

[[ template "common/vault.policies" $n ]]
[[ template "common/file_env" $n ]]

      # Env to access Nomad API
      template {
        data = <<_EOT
NOMAD_TOKEN={{ with secret "nomad/creds/nomad-vector-logger[[ .consul.suffix ]]" }}{{ .Data.secret_id }}{{ end }}
NOMAD_ADDR=https://localhost:4646
NOMAD_CLIENT_CERT=/secrets/nomad.bundle.pem
NOMAD_CLIENT_KEY=/secrets/nomad.bundle.pem
NOMAD_CACERT=/local/nomad.ca.pem
_EOT
        destination = "secrets/.nomad-vector-logger.env"
        perms = 400
        env = true
      }

      # The main configuration file for nomad-vector-logger
      template {
        data = <<_EOT
[[ template "monitoring/agent/nomad-vector-logger.toml" $n ]]
_EOT
        destination = "local/nomad-vector-logger.toml"
      }

      # Disable the default nomad.toml template
      template {
        data = "# Disable the default toml template"
        destination = "local/template/nomad.toml"
      }

      # The vector configuration template used to generate the vector conf
      template {
        data = <<_EOT
[[  template "monitoring/agent/vector-template.yml" $n ]]
_EOT
        destination = "local/template/nomad.yml"
        # {{ }} is used by the template, so prevent consul-template to interprete it
        left_delimiter  = "{{{"
        right_delimiter = "}}}"
      }

      # Get a client cert for the Nomad API
      template {
        data        = <<_EOT
{{- with pkiCert "pki/nomad/issue/nomad-vector-logger[[ .consul.suffix ]]"
                 "common_name=nomad-vector-logger[[ .consul.suffix ]].nomad.[[ .consul.domain ]]"
                 "ttl=72h" }}
{{ .Cert }}
{{ .Key }}
{{- end }}
_EOT
        destination = "secrets/nomad.bundle.pem"
        uid         = 3987
        perms       = "0400"
      }

      # The CA chain to validate Nomad certificates
      template {
        data        = <<_EOT
{{ with secret "pki/nomad/cert/ca_chain" }}{{ .Data.ca_chain }}{{ end }}
_EOT
        destination = "local/nomad.ca.pem"
      }

      # The main config file
      volume_mount {
        volume      = "nomad"
        destination = "/nomad"
        read_only   = true
      }

[[ template "common/resources" $n ]]
    }

    # Nomad Vector Logger can take a few seconds to generate the initial configuration file
    # This task ensure the file exists before vector is started (to prevent an error as the
    # transform_nomad_alloc_* sources won't have anything before the file exists)
    task "wait-for-vector-conf" {

      driver = "[[ $c.nomad.driver ]]"

      config {
        image   = "busybox:latest"
        command = "sh"
        args = [
          "-c",
          "echo 'Waiting for config ffile /alloc/data/vector_conf/nomad.yml to be generated'; until ls /alloc/data/vector_conf/nomad.yml >/dev/null 2>&1; do echo '.'; sleep 2; done"
        ]
      }

      lifecycle {
        hook = "prestart"
      }

      # The task will shutdown once the config is available, so just
      # allocate very few resources
      resources {
        cpu    = 10
        memory = 10
      }
    }

    # The main vector task, which will read logs using the config file generated by Nomad Vector Logger
    task "vector" {

      driver = "[[ $c.nomad.driver ]]"
      leader = true

      config {
        image       = "[[ $c.image ]]"
        userns_mode = "host"
        args = [
          "--watch-config",
          "--config", "/local/vector.yml",
          "--config-dir", "/alloc/data/vector_conf"
        ]
      }

[[ template "common/vault.policies" $c ]]

      env {
        NODE_UNIQUE_NAME = "${node.unique.name}"
      }

[[ template "common/metrics_cert" $c ]]

      template {
        data            =<<_EOT
[[ template "monitoring/agent/vector.yml" $c ]]
_EOT
        destination     = "local/vector.yml"
        left_delimiter  = "{{{"
        right_delimiter = "}}}"

        wait {
          min = "5s"
          max = "30s"
        }
      }

      volume_mount {
        volume      = "nomad"
        destination = "/nomad"
        read_only   = true
      }

      volume_mount {
        volume      = "data"
        destination = "/data"
        read_only   = false
      }

[[ template "common/resources" $c ]]
    }
  }
}