From d24d1230f552463d37e1a604c628e59ed719fd45 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Fri, 5 Jan 2024 14:06:26 +0100 Subject: [PATCH] Add rendered example --- example/LICENSE | 9 + example/README.md | 10 + .../config/service-defaults/immich-ml.hcl | 3 + .../consul/config/service-defaults/immich.hcl | 3 + .../config/service-intentions/immich-ml.hcl | 15 + .../config/service-intentions/immich.hcl | 15 + example/immich.nomad.hcl | 322 ++++++++++++++++++ example/init/vault-database | 12 + example/prep.d/10-mv-conf.sh | 19 ++ example/vault/policies/immich.hcl | 7 + 10 files changed, 415 insertions(+) create mode 100644 example/LICENSE create mode 100644 example/README.md create mode 100644 example/consul/config/service-defaults/immich-ml.hcl create mode 100644 example/consul/config/service-defaults/immich.hcl create mode 100644 example/consul/config/service-intentions/immich-ml.hcl create mode 100644 example/consul/config/service-intentions/immich.hcl create mode 100644 example/immich.nomad.hcl create mode 100755 example/init/vault-database create mode 100755 example/prep.d/10-mv-conf.sh create mode 100644 example/vault/policies/immich.hcl diff --git a/example/LICENSE b/example/LICENSE new file mode 100644 index 0000000..df8dfd2 --- /dev/null +++ b/example/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2023 nomad + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..0000bd2 --- /dev/null +++ b/example/README.md @@ -0,0 +1,10 @@ +# Immich + +Self-hosted photo and video backup solution directly from your mobile phone. + +# Installation +You need to ensure your postgres server has the correct database created (immich, or whatever value you set as instance). This database must have 2 extensions created +``` +CREATE EXTENSION cube; +CREATE EXTENSION earthdistance; +``` diff --git a/example/consul/config/service-defaults/immich-ml.hcl b/example/consul/config/service-defaults/immich-ml.hcl new file mode 100644 index 0000000..113bf3e --- /dev/null +++ b/example/consul/config/service-defaults/immich-ml.hcl @@ -0,0 +1,3 @@ +Kind = "service-defaults" +Name = "immich-ml" +Protocol = "http" diff --git a/example/consul/config/service-defaults/immich.hcl b/example/consul/config/service-defaults/immich.hcl new file mode 100644 index 0000000..e7940cf --- /dev/null +++ b/example/consul/config/service-defaults/immich.hcl @@ -0,0 +1,3 @@ +Kind = "service-defaults" +Name = "immich" +Protocol = "http" diff --git a/example/consul/config/service-intentions/immich-ml.hcl b/example/consul/config/service-intentions/immich-ml.hcl new file mode 100644 index 0000000..4751011 --- /dev/null +++ b/example/consul/config/service-intentions/immich-ml.hcl @@ -0,0 +1,15 @@ +Kind = "service-intentions" +Name = "immich-ml" +Sources = [ + { + Name = "immich" + Permissions = [ + { + Action = "allow" + HTTP { + PathPrefix = "/" + } + } + ] + } +] diff --git a/example/consul/config/service-intentions/immich.hcl b/example/consul/config/service-intentions/immich.hcl new file mode 100644 index 0000000..db2e8d6 --- /dev/null +++ b/example/consul/config/service-intentions/immich.hcl @@ -0,0 +1,15 @@ +Kind = "service-intentions" +Name = "immich" +Sources = [ + { + Name = "traefik" + Permissions = [ + { + Action = "allow" + HTTP { + PathPrefix = "/" + } + } + ] + } +] diff --git a/example/immich.nomad.hcl b/example/immich.nomad.hcl new file mode 100644 index 0000000..d01aee8 --- /dev/null +++ b/example/immich.nomad.hcl @@ -0,0 +1,322 @@ +job "immich" { + + + datacenters = ["dc1"] + + + group "immich" { + + network { + mode = "bridge" + } + + + volume "data" { + source = "immich-data" + type = "csi" + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + + service { + name = "immich" + port = 3001 + + connect { + sidecar_service { + proxy { + upstreams { + destination_name = "immich-ml" + local_bind_port = 3003 + } + upstreams { + destination_name = "postgres" + local_bind_port = 5432 + } + } + } + sidecar_task { + resources { + cpu = 50 + memory = 64 + } + + } + } + + + check { + type = "http" + path = "/api/server-info/ping" + expose = true + interval = "30s" + timeout = "15s" + check_restart { + limit = 10 + grace = "300s" + } + } + + tags = [ + "traefik.enable=true", + + # Define a middleware to set custom CSP headers + "traefik.http.middlewares.immich-headers.headers.contentsecuritypolicy=connect-src 'self' https://maputnik.github.io https://*.cofractal.com https://fonts.openmaptiles.org;default-src 'self';font-src 'self' data:;img-src 'self' data: blob:;script-src 'self' 'unsafe-inline' 'unsafe-eval';style-src 'self' 'unsafe-inline';worker-src 'self' blob:;", + "traefik.http.middlewares.immich-headers.headers.customrequestheaders.X-Forwarded-Proto=https", + # We use a distinct routers for /share so we can apply different middlewares (eg, /share is public while everything else is private) + "traefik.http.routers.immich-share.rule=Host(`immich.example.org`) && PathPrefix(`/share/`)", + "traefik.http.routers.immich-share.entrypoints=https", + "traefik.http.routers.immich-share.middlewares=immich-headers,security-headers@file,hsts@file,compression@file", + + # Main app router + "traefik.http.routers.immich.rule=Host(`immich.example.org`)", + "traefik.http.routers.immich.entrypoints=https", + "traefik.http.routers.immich.middlewares=immich-headers,security-headers@file,hsts@file,compression@file", + ] + } + + # Wait for external services to be ready before continuing + # wait for required services tp be ready before starting the main task + task "wait-for" { + + driver = "docker" + user = 1053 + + config { + image = "danielberteaud/wait-for:24.1-1" + readonly_rootfs = true + pids_limit = 20 + } + + lifecycle { + hook = "prestart" + } + + env { + SERVICE_0 = "master.postgres.service.consul" + SERVICE_1 = "immich-ml.service.consul" + } + + resources { + cpu = 10 + memory = 10 + memory_max = 30 + } + } + + + + # The main immich API server + task "server" { + driver = "docker" + leader = true + # Run as an unprivileged user + user = 3001 + + config { + image = "ghcr.io/immich-app/immich-server:v1.91.4" + readonly_rootfs = true + command = "start.sh" + args = ["immich"] + pids_limit = 100 + } + + vault { + policies = ["immich"] + env = false + disable_file = true + } + + env { + REDIS_HOSTNAME = "127.0.0.1" + IMMICH_MEDIA_LOCATION = "/data" + } + + + # Use a template block instead of env {} so we can fetch values from vault + template { + data = <<_EOT +DB_URL={{ with secret "database/creds/immich" }}postgres://{{ .Data.username }}:{{ urlquery .Data.password }}@localhost:5432/immich{{ end }} +LANG=fr_FR.utf8 +NODE_OPTIONS=--max-old-space-size={{ env "NOMAD_MEMORY_LIMIT" }} +TZ=Europe/Paris +_EOT + destination = "secrets/.env" + perms = 400 + env = true + } + + + volume_mount { + volume = "data" + destination = "/data" + } + + resources { + cpu = 300 + memory = 320 + memory_max = 512 + } + + + } + + + + # microservices is tha task worker, doing all the processing async + task "microservices" { + driver = "docker" + # Run as an unpriviliged user + user = 3001 + + config { + image = "ghcr.io/immich-app/immich-server:v1.91.4" + readonly_rootfs = true + command = "start.sh" + args = ["microservices"] + pids_limit = 100 + } + + vault { + policies = ["immich"] + env = false + disable_file = true + } + + env { + REDIS_HOSTNAME = "127.0.0.1" + IMMICH_MEDIA_LOCATION = "/data" + } + + + # Use a template block instead of env {} so we can fetch values from vault + template { + data = <<_EOT +DB_URL={{ with secret "database/creds/immich" }}postgres://{{ .Data.username }}:{{ urlquery .Data.password }}@localhost:5432/immich{{ end }} +LANG=fr_FR.utf8 +NODE_OPTIONS=--max-old-space-size={{ env "NOMAD_MEMORY_LIMIT" }} +TZ=Europe/Paris +_EOT + destination = "secrets/.env" + perms = 400 + env = true + } + + + volume_mount { + volume = "data" + destination = "/data" + } + + resources { + cpu = 500 + memory = 1024 + memory_max = 1536 + } + + } + + # vim: syntax=hcl + # This is a generic, small and reusable redis task + # It provides no data persistance + + task "redis" { + driver = "docker" + user = 6379 + + lifecycle { + hook = "prestart" + sidecar = true + } + + config { + image = "redis:alpine" + readonly_rootfs = true + args = ["/local/redis.conf"] + } + + template { + data = <<_EOT +bind 127.0.0.1 +maxmemory {{ env "NOMAD_MEMORY_LIMIT" | parseInt | subtract 5 }}mb +databases 1 +save "" +appendonly no +_EOT + destination = "local/redis.conf" + } + + resources { + cpu = 20 + memory = 64 + } + } + + + } + # Used for face recognition, tags etc. + group "machine-learning" { + + network { + mode = "bridge" + } + + + volume "ml" { + source = "immich-ml" + type = "csi" + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + + service { + name = "immich-ml" + port = 3003 + connect { + sidecar_service { + } + sidecar_task { + resources { + cpu = 50 + memory = 64 + } + + } + } + + } + + + task "machine-learning" { + driver = "docker" + user = 3001 + + config { + image = "ghcr.io/immich-app/immich-machine-learning:v1.91.4" + readonly_rootfs = true + pids_limit = 200 + } + + env { + TMPDIR = "/local" + MPLCONFIGDIR = "/local" + MACHINE_LEARNING_HOST = "127.0.0.1" + + + } + + volume_mount { + volume = "ml" + destination = "/cache" + } + + resources { + cpu = 1024 + memory = 1536 + } + + } + } +} diff --git a/example/init/vault-database b/example/init/vault-database new file mode 100755 index 0000000..bf75d1c --- /dev/null +++ b/example/init/vault-database @@ -0,0 +1,12 @@ +#!/bin/sh + +set -euo pipefail + +vault write database/roles/immich \ + db_name="postgres" \ + creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \ + GRANT \"immich\" TO \"{{name}}\"; \ + ALTER ROLE \"{{name}}\" SET role = \"immich\"" \ + default_ttl="12h" \ + max_ttl="720h" + diff --git a/example/prep.d/10-mv-conf.sh b/example/prep.d/10-mv-conf.sh new file mode 100755 index 0000000..58cf3bc --- /dev/null +++ b/example/prep.d/10-mv-conf.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + + + +if [ "immich" != "immich" ]; then + for DIR in vault consul nomad; do + if [ -d output/${DIR} ]; then + for FILE in $(find output/${DIR} -name "*immich*.hcl" -type f); do + NEW_FILE=$(echo "${FILE}" | sed -E "s/immich/immich/g") + mv "${FILE}" "${NEW_FILE}" + done + fi + done +fi + + + diff --git a/example/vault/policies/immich.hcl b/example/vault/policies/immich.hcl new file mode 100644 index 0000000..112e1a7 --- /dev/null +++ b/example/vault/policies/immich.hcl @@ -0,0 +1,7 @@ +path "database/creds/immich" { + capabilities = ["read"] +} + +path "kv/data/service/immich" { + capabilities = ["read"] +}