Import job

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "common"]
+	path = deps/common
+	url = https://git.lapiole.org/nomad/common.git
+	branch = master

README.md

@@ -0,0 +1,11 @@
+# On TrueNAS
+- Turn iSCSI service on
+- Create a portal and an initiator group (note their ID)
+- Create an API key
+
+# On the cluster
+- Store the API key in vault
+```
+vault kv put kv/service/democratic-csi truenas_api_key=XXXXX
+```
+

chown.nomad.hcl

@@ -0,0 +1,47 @@
+job "volume-chown" {
+  datacenters = ["dc1"]
+  type        = "batch"
+
+  group "volume-chown" {
+
+    volume "volume" {
+      type            = "csi"
+      source          = var.vol
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+    }
+
+    task "volume-chown" {
+      driver = "docker"
+      config {
+        image       = "busybox:latest"
+        userns_mode = "host"
+        command     = "chown"
+        args = [
+          "-R",
+          var.owner,
+          "/data"
+        ]
+      }
+
+      volume_mount {
+        volume      = "volume"
+        destination = "/data"
+      }
+
+      resources {
+        cpu    = 10
+        memory = 128
+      }
+    }
+  }
+}
+
+variable "vol" {
+  type    = string
+  default = ""
+}
+variable "owner" {
+  type    = string
+  default = "100000:100000"
+}

controller.nomad.hcl

@@ -0,0 +1,67 @@
+job "democratic-csi-controller" {
+
+  datacenters = [[ .nomad.datacenters | toJSON ]]
+
+[[- range $proto := slice "iscsi" "nfs" ]]
+  group "[[ $proto ]]" {
+
+    count = [[ .democratic_csi.controller.count ]]
+
+    task "iscsi-controller" {
+
+      driver = [[ .democratic_csi.controller.driver | toJSON ]]
+
+      env {
+        NODE_EXTRA_CA_CERTS   = "/local/ca.crt"
+      }
+
+      vault {
+        policies = ["democratic-csi[[ .env.suffix ]]"]
+      }
+
+      config {
+        image = [[ .democratic_csi.image | toJSON ]]
+
+        args = [
+          "--csi-version=1.5.0",
+          "--csi-name=org.democratic-csi.[[ $proto ]]",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock"
+        ]
+
+        network_mode = "host"
+        privileged   = true
+        userns_mode  = "host"
+      }
+
+      template {
+        data        =<<_EOF
+[[ tmpl.Exec (printf "democaratic_csi/%s/controller.yml.tpl" $proto) . ]]
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      # Load vault root CA into the trust store
+      template {
+        data = <<-EOF
+          {{ with secret "[[ .env.suffix ]]pki/root/cert/ca" }}{{ .Data.certificate }}{{ end }}
+        EOF
+        destination = "local/ca.crt"
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.[[ $proto ]]"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+[[ template "common/resources.tpl" .democratic_csi.controller.resources ]]
+    }
+  }
+[[- end ]]
+}
+
+# vim: syntax=hcl

deps/common

@@ -0,0 +1 @@
+Subproject commit 33cfb1fe0c98b3c6437dab509184946cb8551927

images/democratic-csi/Dockerfile

@@ -0,0 +1,8 @@
+ARG CSI_VERSION=1.8.3
+
+FROM docker.io/democraticcsi/democratic-csi:v${CSI_VERSION}
+MAINTAINER Daniel Berteaud <dbd@ehtrace.com>
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

images/democratic-csi/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+# Ensure mount is owned by root. It could be another owner
+# if userns are enabled at the Docker daemon level, but we run this
+# container in the host userns. In this case, root owned files will appear
+# as owned by, eg 100000. And as mount / umount are setuid, it'll drop privileges
+# to this user account, preventing to mount
+
+chown root:root /bin/mount /bin/umount /sbin/mount.nfs
+exec ./bin/democratic-csi $@

node.nomad.hcl

@@ -0,0 +1,89 @@
+job "democratic-csi-node" {
+
+  datacenters = [[ .nomad.datacenters | toJSON ]]
+  type        = "system"
+
+  # Run on all nodes, not matter which pool they belong
+  node_pool = "all"
+
+[[ range $proto := slice "iscsi" "nfs" ]]
+  group "[[ $proto ]]" {
+
+[[- if eq $proto "iscsi" ]]
+    volume "host_root" {
+      type      = "host"
+      read_only = false
+      source    = "host_root"
+    }
+
+    volume "host_run_udev" {
+      type      = "host"
+      read_only = true
+      source    = "host_run_udev"
+    }
+[[- end ]]
+
+    task "[[ $proto ]]-node" {
+
+      driver = [[ democfratic_csi.node.driver | toJSON ]]
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+        [[ template "common/env.tpl" . ]]
+      }
+
+      config {
+        image = [[ .democratic_csi.image | toJSON ]]
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.[[ $proto ]]",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        userns_mode  = "host"
+        privileged   = true
+        ipc_mode     = "host"
+        network_mode = "host"
+      }
+
+      template {
+        data        =<<_EOF
+[[ tmpl.Exec (printf "democratic_csi/%s/node.yml.tpl" $proto) . ]]
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.iscsi"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+[[- if eq $proto "iscsi" ]]
+      volume_mount {
+        volume           = "host_root"
+        destination      = "/host"
+        propagation_mode = "bidirectional"
+      }
+
+      volume_mount {
+        volume      = "host_run_udev"
+        destination = "/run/udev"
+      }
+[[- end ]]
+
+[[ template "common/resources.tpl" .democratic_csi.node.resources ]]
+    }
+[[- end ]]
+  }
+}
+
+# vim: syntax=hcl

templates/iscsi/controller.yml.tpl

@@ -0,0 +1,42 @@
+driver: freenas-api-iscsi
+
+instance_id:
+
+[[- $url := urlParse .democratic_csi.controller.truenas.api.url ]]
+httpConnection:
+  protocol: [[ $url.Scheme ]]
+  host: [[ $url.Host ]]
+  port: [[ $url.Port ]]
+  apiKey: {{ with secret "[[ .vault.prefix ]]kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }}
+  allowInsecure: [[ .democratic_csi.controllers.truenas.api.insecure | ternary "true" "false" ]]
+  apiVersion: 2
+
+zfs:
+  datasetParentName: [[ .democratic_csi.controllers.iscsi.zfs.parent ]]
+  detachedSnapshotsDatasetParentName: [[ .democratic_csi.controllers.iscsi.zfs.snap_parent ]]
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: [[ .democratic_csi.controllers.iscsi.zfs.reservation ]]
+  zvolBlocksize: [[ .democratic_csi.controllers.iscsi.zfs.reservation ]]
+
+iscsi:
+  targetPortals:
+[[- range $idx, $portal := .democratic_csi.controllers.iscsi.portals ]]
+    - [[ $portal ]][[ end ]]
+  interface:
+
+  namePrefix: ""
+  nameSuffix: ""
+
+  targetGroups:
+    - targetGroupPortalGroup: [[ .democratic_csi.controllers.iscsi.target_group.portal_group ]]
+      targetGroupInitiatorGroup: [[ .democratic_csi.controllers.iscsi.target_group.initiator_group ]]
+      targetGroupAuthType: [[ .democratic_csi.controller.iscsi.target_group.auth_type ]]
+      targetGroupAuthGroup: [[ .democratic_csi.controller.iscsi.target_group.auth_group ]]
+
+  extentInsecureTpc: true
+  extentXenCompat: false
+  extentDisablePhysicalBlocksize: false
+  extentBlocksize: 512
+  extentRpm: "SSD"
+  extentAvailThreshold: 0

templates/iscsi/node.yml.tpl

@@ -0,0 +1,4 @@
+---
+driver: freenas-api-iscsi
+instance_id:
+

templates/nfs/controller.yml.tpl

@@ -0,0 +1,33 @@
+driver: freenas-api-nfs
+
+instance_id:
+
+[[- $url := urlParse .democratic_csi.controller.truenas.api.url ]]
+httpConnection:
+  protocol: [[ $url.Scheme ]]
+  host: [[ $url.Host ]]
+  port: [[ $url.Port ]]
+  apiKey: {{ with secret "[[ .vault.prefix ]]kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }}
+  allowInsecure: [[ .democratic_csi.controllers.truenas.api.insecure | ternary "true" "false" ]]
+  apiVersion: 2
+
+zfs:
+  datasetParentName: [[ .democratic_csi.controller.nfs.zfs.parent ]]
+  detachedSnapshotsDatasetParentName: [[ .democratic_csi.controller.nfs.zfs.snap_parent ]]
+  datasetEnableQuotas: [[ .democratic_csi.controller.nfs.zfs.quota ]]
+  datasetEnableReservation: [[ .democratic_csi.controller.nfs.zfs.reservation ]]
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 0
+  datasetPermissionsGroup: 0
+
+nfs:
+  shareHost: [[ .democratic_csi.controller.nfs.server ]]
+  shareAlldirs: false
+  shareAllowedHosts: []
+  shareAllowedNetworks:
+[[- range $idx, $cidr := .democratic_csi.controller.nfs.allowed_networks ]]
+    - [[ $cidr ]][[ end ]]
+  shareMaprootUser: root
+  shareMaprootGroup: root
+  shareMapallUser: ""
+  shareMapallGroup: ""

templates/nfs/node.yml.tpl

@@ -0,0 +1,4 @@
+---
+driver: freenas-api-nfs
+instance_id:
+

variables.yml

@@ -0,0 +1,67 @@
+---
+
+democratic_csi:
+  # The Docker image to use
+  image: danielberteaud/democratic-csi:1.8.3-4
+
+  # Settings for controllers
+  controller:
+
+    # Number of controler instances
+    count: 1
+
+    # Driver (docker or podman)
+    driver: docker
+
+    # Resource allocation for controllers
+    resources:
+      cpu: 100
+      memory: 192
+
+    truenas:
+      api:
+        # The URL to reach truenas API
+        url: https://truenas.example.org:443
+        # Set to true to disable certificate verification
+        # Which is of course not recommanded
+        insecure: False
+        # Note: the API Key is stored in vault, in kv/service/democratic-csi in the truenas_api_key prop
+
+      iscsi:
+
+        zfs:
+          parent: zpool/csi/iscsi
+          snap_parent: zpool/csi/iscsisnap
+          reservation: false
+          blocksize: 16K
+
+        portals:
+          - 10.99.3.27:3260
+
+        target_group:
+          portal_group: 1
+          initialor_group: 1
+          auth_type: None
+          auth_group: ""
+
+      nfs:
+        zfs:
+          parent: zpool/csi/nfs
+          snap_parent: zpool/csi/nfssnap
+          quota: False
+          resrvation: False
+          server: 10.99.3.27
+          allowed_networks:
+            - 10.99.9.0/24
+  
+  # Settings for node plugins
+  node:
+
+    # Driver (docker or podman)
+    driver: docker
+
+    # Resource allocation
+    resources:
+      cpu: 100
+      memory: 192
+

vault/policies/democratic-csi.hcl

@@ -0,0 +1,3 @@
+path "[[ .vault.prefix ]]kv/data/service/democratic-csi" {
+  capabilities = ["read"]
+}

volumes/iscsi-truenas-api.hcl

@@ -0,0 +1,18 @@
+type = "csi"
+id = "csi-volume-iscsi"
+name = "csi-volume-iscsi"
+plugin_id = "org.democratic-csi.iscsi"
+capacity_min = "1GiB"
+capacity_max = "1GiB"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  # ext4|xfs
+  # default is ext4 when left unset
+  fs_type     = "xfs"
+  mount_flags = ["noatime"]
+}

volumes/nfs-truenas-api.hcl

@@ -0,0 +1,15 @@
+type = "csi"
+id = "csi-volume-nfs"
+name = "csi-volume-nfs"
+plugin_id = "org.democratic-csi.nfs"
+capacity_min = "1GiB"
+capacity_max = "2GiB"
+
+capability {
+  access_mode     = "multi-node-multi-writer"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  mount_flags = ["noatime"]
+}