Add rendered example

example/README.md

@@ -0,0 +1,15 @@
+# Democratic CSI using FreeNAS API
+To have this running, you first need to configure your truenas scale
+
+- Turn iSCSI service on
+- Create a portal and an initiator group (note their ID)
+- Create an API key
+
+Then, store the API key in vault
+
+- Store the API key in vault
+```
+vault kv put kv/service/democratic-csi truenas_api_key=XXXXX
+```
+
+

example/chown.nomad.hcl

@@ -0,0 +1,47 @@
+job "volume-chown" {
+  datacenters = ["dc1"]
+  type        = "batch"
+
+  group "volume-chown" {
+
+    volume "volume" {
+      type            = "csi"
+      source          = var.vol
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
+    }
+
+    task "volume-chown" {
+      driver = "docker"
+      config {
+        image       = "busybox:latest"
+        userns_mode = "host"
+        command     = "chown"
+        args = [
+          "-R",
+          var.owner,
+          "/data"
+        ]
+      }
+
+      volume_mount {
+        volume      = "volume"
+        destination = "/data"
+      }
+
+      resources {
+        cpu    = 10
+        memory = 128
+      }
+    }
+  }
+}
+
+variable "vol" {
+  type    = string
+  default = ""
+}
+variable "owner" {
+  type    = string
+  default = "100000:100000"
+}

example/controller.nomad.hcl

@@ -0,0 +1,223 @@
+job "democratic-csi-controller" {
+  datacenters = ["dc1"]
+
+
+  group "iscsi-controller" {
+
+    count = 1
+
+    restart {
+      interval = "5m"
+      attempts = 30
+      delay    = "10s"
+      mode     = "delay"
+    }
+
+    task "iscsi-controller" {
+
+      driver = "docker"
+
+      env {
+        NODE_EXTRA_CA_CERTS = "/local/ca.crt"
+
+        LANG = "fr_FR.utf8"
+        TZ   = "Europe/Paris"
+
+      }
+
+      vault {
+        policies = ["democratic-csi"]
+      }
+
+      config {
+        image = "danielberteaud/democratic-csi:1.8.4-1"
+
+        args = [
+          "--csi-version=1.5.0",
+          "--csi-name=org.democratic-csi.iscsi",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock"
+        ]
+
+        network_mode = "host"
+        privileged   = true
+        userns_mode  = "host"
+      }
+
+      template {
+        data        = <<_EOF
+driver: freenas-api-iscsi
+
+instance_id:
+httpConnection:
+  protocol: https
+  host: truenas.example.org:443
+  port: 443
+  apiKey: {{ with secret "kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }}
+  allowInsecure: false
+  apiVersion: 2
+
+zfs:
+  datasetParentName: zpool/csi/iscsi
+  detachedSnapshotsDatasetParentName: zpool/csi/iscsisnap
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize: false
+
+iscsi:
+  targetPortals:
+    - 10.99.3.27:3260
+  interface:
+
+  namePrefix: ""
+  nameSuffix: ""
+
+  targetGroups:
+    - targetGroupPortalGroup: 1
+      targetGroupInitiatorGroup: 1
+      targetGroupAuthType: None
+      targetGroupAuthGroup: 
+
+  extentInsecureTpc: true
+  extentXenCompat: false
+  extentDisablePhysicalBlocksize: false
+  extentBlocksize: 512
+  extentRpm: "SSD"
+  extentAvailThreshold: 0
+
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      # Load vault root CA into the trust store
+      template {
+        data        = <<-EOF
+          {{ with secret "pki/root/cert/ca" }}{{ .Data.certificate }}{{ end }}
+        EOF
+        destination = "local/ca.crt"
+      }
+
+      csi_plugin {
+        id        = "org.democratic-csi.iscsi"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu        = 100
+        memory     = 128
+        memory_max = 192
+      }
+
+    }
+  }
+
+  group "nfs-controller" {
+
+    count = 1
+
+    restart {
+      interval = "5m"
+      attempts = 30
+      delay    = "10s"
+      mode     = "delay"
+    }
+
+    task "nfs-controller" {
+
+      driver = "docker"
+
+      env {
+        NODE_EXTRA_CA_CERTS = "/local/ca.crt"
+
+        LANG = "fr_FR.utf8"
+        TZ   = "Europe/Paris"
+
+      }
+
+      vault {
+        policies = ["democratic-csi"]
+      }
+
+      config {
+        image = "danielberteaud/democratic-csi:1.8.4-1"
+
+        args = [
+          "--csi-version=1.5.0",
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock"
+        ]
+
+        network_mode = "host"
+        privileged   = true
+        userns_mode  = "host"
+      }
+
+      template {
+        data        = <<_EOF
+driver: freenas-api-nfs
+
+instance_id:
+httpConnection:
+  protocol: https
+  host: truenas.example.org:443
+  port: 443
+  apiKey: {{ with secret "kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }}
+  allowInsecure: false
+  apiVersion: 2
+
+zfs:
+  datasetParentName: zpool/csi/nfs
+  detachedSnapshotsDatasetParentName: zpool/csi/nfssnap
+  datasetEnableQuotas: false
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 0
+  datasetPermissionsGroup: 0
+
+nfs:
+  shareHost: 10.99.3.27
+  shareAlldirs: false
+  shareAllowedHosts: []
+  shareAllowedNetworks:
+    - 10.99.9.0/24
+  shareMaprootUser: root
+  shareMaprootGroup: root
+  shareMapallUser: ""
+  shareMapallGroup: ""
+
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      # Load vault root CA into the trust store
+      template {
+        data        = <<-EOF
+          {{ with secret "pki/root/cert/ca" }}{{ .Data.certificate }}{{ end }}
+        EOF
+        destination = "local/ca.crt"
+      }
+
+      csi_plugin {
+        id        = "org.democratic-csi.nfs"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu        = 100
+        memory     = 128
+        memory_max = 192
+      }
+
+    }
+  }
+}
+
+# vim: syntax=hcl

example/images/democratic-csi/Dockerfile

@@ -0,0 +1,8 @@
+ARG CSI_VERSION=1.8.4
+
+FROM docker.io/democraticcsi/democratic-csi:v${CSI_VERSION}
+MAINTAINER Daniel Berteaud <dbd@ehtrace.com>
+
+COPY root/ /
+
+ENTRYPOINT ["/entrypoint.sh"]

example/images/democratic-csi/root/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+# Ensure mount is owned by root. It could be another owner
+# if userns are enabled at the Docker daemon level, but we run this
+# container in the host userns. In this case, root owned files will appear
+# as owned by, eg 100000. And as mount / umount are setuid, it'll drop privileges
+# to this user account, preventing to mount
+
+chown root:root /bin/mount /bin/umount /sbin/mount.nfs
+exec ./bin/democratic-csi $@

example/node.nomad.hcl

@@ -0,0 +1,165 @@
+job "democratic-csi-node" {
+  type        = "system"
+  datacenters = ["dc1"]
+
+
+  group "iscsi-node" {
+
+    restart {
+      interval = "5m"
+      attempts = 30
+      delay    = "10s"
+      mode     = "delay"
+    }
+    volume "host_root" {
+      source = "host_root"
+      type   = "host"
+    }
+    volume "host_run_udev" {
+      source    = "host_run_udev"
+      type      = "host"
+      read_only = true
+    }
+
+
+    task "iscsi-node" {
+
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+
+        LANG = "fr_FR.utf8"
+        TZ   = "Europe/Paris"
+
+      }
+
+      config {
+        image = "danielberteaud/democratic-csi:1.8.4-1"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.iscsi",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        userns_mode  = "host"
+        privileged   = true
+        ipc_mode     = "host"
+        network_mode = "host"
+      }
+
+      template {
+        data        = <<_EOF
+---
+driver: freenas-api-iscsi
+instance_id:
+
+
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.iscsi"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+      volume_mount {
+        volume           = "host_root"
+        destination      = "/host"
+        propagation_mode = "bidirectional"
+      }
+
+      volume_mount {
+        volume      = "host_run_udev"
+        destination = "/run/udev"
+      }
+
+      resources {
+        cpu        = 100
+        memory     = 128
+        memory_max = 192
+      }
+
+    }
+  }
+
+  group "nfs-node" {
+
+    restart {
+      interval = "5m"
+      attempts = 30
+      delay    = "10s"
+      mode     = "delay"
+    }
+
+    task "nfs-node" {
+
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+
+        LANG = "fr_FR.utf8"
+        TZ   = "Europe/Paris"
+
+      }
+
+      config {
+        image = "danielberteaud/democratic-csi:1.8.4-1"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=/secrets/config.yml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        userns_mode  = "host"
+        privileged   = true
+        ipc_mode     = "host"
+        network_mode = "host"
+      }
+
+      template {
+        data        = <<_EOF
+---
+driver: freenas-api-nfs
+instance_id:
+
+
+_EOF
+        destination = "secrets/config.yml"
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu        = 100
+        memory     = 128
+        memory_max = 192
+      }
+
+    }
+  }
+}
+
+# vim: syntax=hcl

example/vault/policies/democratic-csi.hcl

@@ -0,0 +1,3 @@
+path "kv/data/service/democratic-csi" {
+  capabilities = ["read"]
+}

example/volumes/iscsi-truenas-api.hcl

@@ -0,0 +1,18 @@
+type = "csi"
+id = "csi-volume-iscsi"
+name = "csi-volume-iscsi"
+plugin_id = "org.democratic-csi.iscsi"
+capacity_min = "1GiB"
+capacity_max = "1GiB"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  # ext4|xfs
+  # default is ext4 when left unset
+  fs_type     = "xfs"
+  mount_flags = ["noatime"]
+}

example/volumes/nfs-truenas-api.hcl

@@ -0,0 +1,15 @@
+type = "csi"
+id = "csi-volume-nfs"
+name = "csi-volume-nfs"
+plugin_id = "org.democratic-csi.nfs"
+capacity_min = "1GiB"
+capacity_max = "2GiB"
+
+capability {
+  access_mode     = "multi-node-multi-writer"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  mount_flags = ["noatime"]
+}