From 74b8604f77ea76d9e1d9947850cb97504f68f821 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Mon, 21 Aug 2023 20:16:35 +0200 Subject: [PATCH] Import job --- .gitmodules | 4 ++ README.md | 11 ++++ chown.nomad.hcl | 47 +++++++++++++++ controller.nomad.hcl | 67 ++++++++++++++++++++++ deps/common | 1 + images/democratic-csi/Dockerfile | 8 +++ images/democratic-csi/entrypoint.sh | 12 ++++ node.nomad.hcl | 89 +++++++++++++++++++++++++++++ templates/iscsi/controller.yml.tpl | 42 ++++++++++++++ templates/iscsi/node.yml.tpl | 4 ++ templates/nfs/controller.yml.tpl | 33 +++++++++++ templates/nfs/node.yml.tpl | 4 ++ variables.yml | 67 ++++++++++++++++++++++ vault/policies/democratic-csi.hcl | 3 + volumes/iscsi-truenas-api.hcl | 18 ++++++ volumes/nfs-truenas-api.hcl | 15 +++++ 16 files changed, 425 insertions(+) create mode 100644 .gitmodules create mode 100644 README.md create mode 100644 chown.nomad.hcl create mode 100644 controller.nomad.hcl create mode 160000 deps/common create mode 100644 images/democratic-csi/Dockerfile create mode 100755 images/democratic-csi/entrypoint.sh create mode 100644 node.nomad.hcl create mode 100644 templates/iscsi/controller.yml.tpl create mode 100644 templates/iscsi/node.yml.tpl create mode 100644 templates/nfs/controller.yml.tpl create mode 100644 templates/nfs/node.yml.tpl create mode 100644 variables.yml create mode 100644 vault/policies/democratic-csi.hcl create mode 100644 volumes/iscsi-truenas-api.hcl create mode 100644 volumes/nfs-truenas-api.hcl diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e18dbaf --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "common"] + path = deps/common + url = https://git.lapiole.org/nomad/common.git + branch = master diff --git a/README.md b/README.md new file mode 100644 index 0000000..185f619 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# On TrueNAS +- Turn iSCSI service on +- Create a portal and an initiator group (note their ID) +- Create an API key + +# On the cluster +- Store the API key in vault +``` +vault kv put kv/service/democratic-csi truenas_api_key=XXXXX +``` + diff --git a/chown.nomad.hcl b/chown.nomad.hcl new file mode 100644 index 0000000..ab6e644 --- /dev/null +++ b/chown.nomad.hcl @@ -0,0 +1,47 @@ +job "volume-chown" { + datacenters = ["dc1"] + type = "batch" + + group "volume-chown" { + + volume "volume" { + type = "csi" + source = var.vol + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + task "volume-chown" { + driver = "docker" + config { + image = "busybox:latest" + userns_mode = "host" + command = "chown" + args = [ + "-R", + var.owner, + "/data" + ] + } + + volume_mount { + volume = "volume" + destination = "/data" + } + + resources { + cpu = 10 + memory = 128 + } + } + } +} + +variable "vol" { + type = string + default = "" +} +variable "owner" { + type = string + default = "100000:100000" +} diff --git a/controller.nomad.hcl b/controller.nomad.hcl new file mode 100644 index 0000000..7ff59be --- /dev/null +++ b/controller.nomad.hcl @@ -0,0 +1,67 @@ +job "democratic-csi-controller" { + + datacenters = [[ .nomad.datacenters | toJSON ]] + +[[- range $proto := slice "iscsi" "nfs" ]] + group "[[ $proto ]]" { + + count = [[ .democratic_csi.controller.count ]] + + task "iscsi-controller" { + + driver = [[ .democratic_csi.controller.driver | toJSON ]] + + env { + NODE_EXTRA_CA_CERTS = "/local/ca.crt" + } + + vault { + policies = ["democratic-csi[[ .env.suffix ]]"] + } + + config { + image = [[ .democratic_csi.image | toJSON ]] + + args = [ + "--csi-version=1.5.0", + "--csi-name=org.democratic-csi.[[ $proto ]]", + "--driver-config-file=/secrets/config.yml", + "--log-level=info", + "--csi-mode=controller", + "--server-socket=/csi/csi.sock" + ] + + network_mode = "host" + privileged = true + userns_mode = "host" + } + + template { + data =<<_EOF +[[ tmpl.Exec (printf "democaratic_csi/%s/controller.yml.tpl" $proto) . ]] +_EOF + destination = "secrets/config.yml" + } + + # Load vault root CA into the trust store + template { + data = <<-EOF + {{ with secret "[[ .env.suffix ]]pki/root/cert/ca" }}{{ .Data.certificate }}{{ end }} + EOF + destination = "local/ca.crt" + } + + csi_plugin { + # must match --csi-name arg + id = "org.democratic-csi.[[ $proto ]]" + type = "controller" + mount_dir = "/csi" + } + +[[ template "common/resources.tpl" .democratic_csi.controller.resources ]] + } + } +[[- end ]] +} + +# vim: syntax=hcl diff --git a/deps/common b/deps/common new file mode 160000 index 0000000..33cfb1f --- /dev/null +++ b/deps/common @@ -0,0 +1 @@ +Subproject commit 33cfb1fe0c98b3c6437dab509184946cb8551927 diff --git a/images/democratic-csi/Dockerfile b/images/democratic-csi/Dockerfile new file mode 100644 index 0000000..523cd42 --- /dev/null +++ b/images/democratic-csi/Dockerfile @@ -0,0 +1,8 @@ +ARG CSI_VERSION=1.8.3 + +FROM docker.io/democraticcsi/democratic-csi:v${CSI_VERSION} +MAINTAINER Daniel Berteaud + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/democratic-csi/entrypoint.sh b/images/democratic-csi/entrypoint.sh new file mode 100755 index 0000000..1696767 --- /dev/null +++ b/images/democratic-csi/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +# Ensure mount is owned by root. It could be another owner +# if userns are enabled at the Docker daemon level, but we run this +# container in the host userns. In this case, root owned files will appear +# as owned by, eg 100000. And as mount / umount are setuid, it'll drop privileges +# to this user account, preventing to mount + +chown root:root /bin/mount /bin/umount /sbin/mount.nfs +exec ./bin/democratic-csi $@ diff --git a/node.nomad.hcl b/node.nomad.hcl new file mode 100644 index 0000000..f31b8cb --- /dev/null +++ b/node.nomad.hcl @@ -0,0 +1,89 @@ +job "democratic-csi-node" { + + datacenters = [[ .nomad.datacenters | toJSON ]] + type = "system" + + # Run on all nodes, not matter which pool they belong + node_pool = "all" + +[[ range $proto := slice "iscsi" "nfs" ]] + group "[[ $proto ]]" { + +[[- if eq $proto "iscsi" ]] + volume "host_root" { + type = "host" + read_only = false + source = "host_root" + } + + volume "host_run_udev" { + type = "host" + read_only = true + source = "host_run_udev" + } +[[- end ]] + + task "[[ $proto ]]-node" { + + driver = [[ democfratic_csi.node.driver | toJSON ]] + + env { + CSI_NODE_ID = "${attr.unique.hostname}" + [[ template "common/env.tpl" . ]] + } + + config { + image = [[ .democratic_csi.image | toJSON ]] + + args = [ + "--csi-version=1.5.0", + # must match the csi_plugin.id attribute below + "--csi-name=org.democratic-csi.[[ $proto ]]", + "--driver-config-file=/secrets/config.yml", + "--log-level=info", + "--csi-mode=node", + "--server-socket=/csi/csi.sock", + ] + + # node plugins must run as privileged jobs because they + # mount disks to the host + userns_mode = "host" + privileged = true + ipc_mode = "host" + network_mode = "host" + } + + template { + data =<<_EOF +[[ tmpl.Exec (printf "democratic_csi/%s/node.yml.tpl" $proto) . ]] +_EOF + destination = "secrets/config.yml" + } + + csi_plugin { + # must match --csi-name arg + id = "org.democratic-csi.iscsi" + type = "node" + mount_dir = "/csi" + } + +[[- if eq $proto "iscsi" ]] + volume_mount { + volume = "host_root" + destination = "/host" + propagation_mode = "bidirectional" + } + + volume_mount { + volume = "host_run_udev" + destination = "/run/udev" + } +[[- end ]] + +[[ template "common/resources.tpl" .democratic_csi.node.resources ]] + } +[[- end ]] + } +} + +# vim: syntax=hcl diff --git a/templates/iscsi/controller.yml.tpl b/templates/iscsi/controller.yml.tpl new file mode 100644 index 0000000..75cc88a --- /dev/null +++ b/templates/iscsi/controller.yml.tpl @@ -0,0 +1,42 @@ +driver: freenas-api-iscsi + +instance_id: + +[[- $url := urlParse .democratic_csi.controller.truenas.api.url ]] +httpConnection: + protocol: [[ $url.Scheme ]] + host: [[ $url.Host ]] + port: [[ $url.Port ]] + apiKey: {{ with secret "[[ .vault.prefix ]]kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }} + allowInsecure: [[ .democratic_csi.controllers.truenas.api.insecure | ternary "true" "false" ]] + apiVersion: 2 + +zfs: + datasetParentName: [[ .democratic_csi.controllers.iscsi.zfs.parent ]] + detachedSnapshotsDatasetParentName: [[ .democratic_csi.controllers.iscsi.zfs.snap_parent ]] + zvolCompression: + zvolDedup: + zvolEnableReservation: [[ .democratic_csi.controllers.iscsi.zfs.reservation ]] + zvolBlocksize: [[ .democratic_csi.controllers.iscsi.zfs.reservation ]] + +iscsi: + targetPortals: +[[- range $idx, $portal := .democratic_csi.controllers.iscsi.portals ]] + - [[ $portal ]][[ end ]] + interface: + + namePrefix: "" + nameSuffix: "" + + targetGroups: + - targetGroupPortalGroup: [[ .democratic_csi.controllers.iscsi.target_group.portal_group ]] + targetGroupInitiatorGroup: [[ .democratic_csi.controllers.iscsi.target_group.initiator_group ]] + targetGroupAuthType: [[ .democratic_csi.controller.iscsi.target_group.auth_type ]] + targetGroupAuthGroup: [[ .democratic_csi.controller.iscsi.target_group.auth_group ]] + + extentInsecureTpc: true + extentXenCompat: false + extentDisablePhysicalBlocksize: false + extentBlocksize: 512 + extentRpm: "SSD" + extentAvailThreshold: 0 diff --git a/templates/iscsi/node.yml.tpl b/templates/iscsi/node.yml.tpl new file mode 100644 index 0000000..a3c7cba --- /dev/null +++ b/templates/iscsi/node.yml.tpl @@ -0,0 +1,4 @@ +--- +driver: freenas-api-iscsi +instance_id: + diff --git a/templates/nfs/controller.yml.tpl b/templates/nfs/controller.yml.tpl new file mode 100644 index 0000000..5f09828 --- /dev/null +++ b/templates/nfs/controller.yml.tpl @@ -0,0 +1,33 @@ +driver: freenas-api-nfs + +instance_id: + +[[- $url := urlParse .democratic_csi.controller.truenas.api.url ]] +httpConnection: + protocol: [[ $url.Scheme ]] + host: [[ $url.Host ]] + port: [[ $url.Port ]] + apiKey: {{ with secret "[[ .vault.prefix ]]kv/service/democratic-csi" }}{{ .Data.data.truenas_api_key }}{{ end }} + allowInsecure: [[ .democratic_csi.controllers.truenas.api.insecure | ternary "true" "false" ]] + apiVersion: 2 + +zfs: + datasetParentName: [[ .democratic_csi.controller.nfs.zfs.parent ]] + detachedSnapshotsDatasetParentName: [[ .democratic_csi.controller.nfs.zfs.snap_parent ]] + datasetEnableQuotas: [[ .democratic_csi.controller.nfs.zfs.quota ]] + datasetEnableReservation: [[ .democratic_csi.controller.nfs.zfs.reservation ]] + datasetPermissionsMode: "0770" + datasetPermissionsUser: 0 + datasetPermissionsGroup: 0 + +nfs: + shareHost: [[ .democratic_csi.controller.nfs.server ]] + shareAlldirs: false + shareAllowedHosts: [] + shareAllowedNetworks: +[[- range $idx, $cidr := .democratic_csi.controller.nfs.allowed_networks ]] + - [[ $cidr ]][[ end ]] + shareMaprootUser: root + shareMaprootGroup: root + shareMapallUser: "" + shareMapallGroup: "" diff --git a/templates/nfs/node.yml.tpl b/templates/nfs/node.yml.tpl new file mode 100644 index 0000000..b3b4ccd --- /dev/null +++ b/templates/nfs/node.yml.tpl @@ -0,0 +1,4 @@ +--- +driver: freenas-api-nfs +instance_id: + diff --git a/variables.yml b/variables.yml new file mode 100644 index 0000000..2a443f3 --- /dev/null +++ b/variables.yml @@ -0,0 +1,67 @@ +--- + +democratic_csi: + # The Docker image to use + image: danielberteaud/democratic-csi:1.8.3-4 + + # Settings for controllers + controller: + + # Number of controler instances + count: 1 + + # Driver (docker or podman) + driver: docker + + # Resource allocation for controllers + resources: + cpu: 100 + memory: 192 + + truenas: + api: + # The URL to reach truenas API + url: https://truenas.example.org:443 + # Set to true to disable certificate verification + # Which is of course not recommanded + insecure: False + # Note: the API Key is stored in vault, in kv/service/democratic-csi in the truenas_api_key prop + + iscsi: + + zfs: + parent: zpool/csi/iscsi + snap_parent: zpool/csi/iscsisnap + reservation: false + blocksize: 16K + + portals: + - 10.99.3.27:3260 + + target_group: + portal_group: 1 + initialor_group: 1 + auth_type: None + auth_group: "" + + nfs: + zfs: + parent: zpool/csi/nfs + snap_parent: zpool/csi/nfssnap + quota: False + resrvation: False + server: 10.99.3.27 + allowed_networks: + - 10.99.9.0/24 + + # Settings for node plugins + node: + + # Driver (docker or podman) + driver: docker + + # Resource allocation + resources: + cpu: 100 + memory: 192 + diff --git a/vault/policies/democratic-csi.hcl b/vault/policies/democratic-csi.hcl new file mode 100644 index 0000000..c80b228 --- /dev/null +++ b/vault/policies/democratic-csi.hcl @@ -0,0 +1,3 @@ +path "[[ .vault.prefix ]]kv/data/service/democratic-csi" { + capabilities = ["read"] +} diff --git a/volumes/iscsi-truenas-api.hcl b/volumes/iscsi-truenas-api.hcl new file mode 100644 index 0000000..514e8f0 --- /dev/null +++ b/volumes/iscsi-truenas-api.hcl @@ -0,0 +1,18 @@ +type = "csi" +id = "csi-volume-iscsi" +name = "csi-volume-iscsi" +plugin_id = "org.democratic-csi.iscsi" +capacity_min = "1GiB" +capacity_max = "1GiB" + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +mount_options { + # ext4|xfs + # default is ext4 when left unset + fs_type = "xfs" + mount_flags = ["noatime"] +} diff --git a/volumes/nfs-truenas-api.hcl b/volumes/nfs-truenas-api.hcl new file mode 100644 index 0000000..237f08a --- /dev/null +++ b/volumes/nfs-truenas-api.hcl @@ -0,0 +1,15 @@ +type = "csi" +id = "csi-volume-nfs" +name = "csi-volume-nfs" +plugin_id = "org.democratic-csi.nfs" +capacity_min = "1GiB" +capacity_max = "2GiB" + +capability { + access_mode = "multi-node-multi-writer" + attachment_mode = "file-system" +} + +mount_options { + mount_flags = ["noatime"] +}