dani
/
ctctl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Various fixes 

Veuillez saisir le message de validation pour vos modifications. Les lignes
This commit is contained in:
Daniel Berteaud 2023-08-24 01:05:06 +02:00
parent bee1de9d32
commit 9e97651df4
4 changed files with 147 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -1,2 +1,2 @@
# ctctl
# ctctl is a tool to work with Nomad / Consul / Vault container orchestration stack

222
ctctl
View File

@ -267,91 +267,85 @@ load_policies(){
kill -INT $$
fi
local DIR="./"
if [ -e ctctl.bundle.conf ]; then
DIR="./output"
fi
if [ -d "${DIR}/vault/policies" ]; then
if [ "$(check_vault_token)" != "1" ]; then
echo "No valid vault token. You have to authenticate first"
kill -INT $$
fi
for PFILE in $(ls ${DIR}/vault/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
echo "Loading vault policy ${PNA}"
replace_conf_var ${PFILE} | vault policy write ${PNAME} -
done
fi
if [ -d "${DIR}/consul/policies" ]; then
if [ "$(check_consul_token)" != "1" ]; then
echo "No valid consul token. You have to authenticate first"
kill -INT $$
fi
CONSUL_CUR_POLICIES=$(consul acl policy list -format=json)
for PFILE in $(ls ${DIR}/consul/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
# Consul do not use the same command to create a new policy and to update an existing one
# so we need to detect if the policy already exists
if [ "$(echo ${CONSUL_CUR_POLICIES} | jq -r '.[] | select(.Name=='\"${PNAME}\"') | .Name')" == "${PNAME}" ]; then
echo "Updating consul policy ${PNAME}"
replace_conf_var ${PFILE} | consul acl policy update -name=${PNAME} -rules=-
else
echo "Adding new consul policy ${PNAME}"
replace_conf_var ${PFILE} | consul acl policy create -name=${PNAME} -rules=-
for DIR in ./output ./; do
if [ -d "${DIR}/vault/policies" ]; then
if [ "$(check_vault_token)" != "1" ]; then
echo "No valid vault token. You have to authenticate first"
kill -INT $$
fi
done
fi
if [ -d "${DIR}/nomad/policies" ]; then
if [ "$(check_nomad_token)" != "1" ]; then
echo "No valid nomad token. You have to authenticate first"
kill -INT $$
for PFILE in $(ls ${DIR}/vault/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
echo "Loading vault policy ${PNA}"
replace_conf_var ${PFILE} | vault policy write ${PNAME} -
done
fi
for PFILE in $(ls ${DIR}nomad/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
echo "Loading Nomad policy ${PNAME}"
replace_conf_var ${PFILE} | nomad acl policy apply ${PNAME} -
done
fi
if [ -d "${DIR}/consul/policies" ]; then
if [ "$(check_consul_token)" != "1" ]; then
echo "No valid consul token. You have to authenticate first"
kill -INT $$
fi
CONSUL_CUR_POLICIES=$(consul acl policy list -format=json)
for PFILE in $(ls ${DIR}/consul/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
# Consul do not use the same command to create a new policy and to update an existing one
# so we need to detect if the policy already exists
if [ "$(echo ${CONSUL_CUR_POLICIES} | jq -r '.[] | select(.Name=='\"${PNAME}\"') | .Name')" == "${PNAME}" ]; then
echo "Updating consul policy ${PNAME}"
replace_conf_var ${PFILE} | consul acl policy update -name=${PNAME} -rules=-
else
echo "Adding new consul policy ${PNAME}"
replace_conf_var ${PFILE} | consul acl policy create -name=${PNAME} -rules=-
fi
done
fi
if [ -d "${DIR}/nomad/policies" ]; then
if [ "$(check_nomad_token)" != "1" ]; then
echo "No valid nomad token. You have to authenticate first"
kill -INT $$
fi
for PFILE in $(ls ${DIR}nomad/policies/*.hcl 2>/dev/null); do
PNAME=$(basename ${PFILE} .hcl)
echo "Loading Nomad policy ${PNAME}"
replace_conf_var ${PFILE} | nomad acl policy apply ${PNAME} -
done
fi
done
}
# Load consul config
load_consul_conf(){
local DIR="./"
if [ -e ctctl.bundle.conf ]; then
DIR="./output"
fi
if [ -d "${DIR}/consul/config" ]; then
if [ "$(check_consul_token)" != "1" ]; then
echo "No valid consul token. You have to authenticate first"
kill -INT $$
fi
# Note : service-defaults should be loaded before the others
# but that should be the case
for FILE in $(ls ${DIR}/consul/config/*.hcl 2>/dev/null); do
echo "Loading consul conf from ${FILE}"
TEMP=$(mktemp)
replace_conf_var ${FILE} > ${TEMP}
consul config write ${TEMP}
rm -f ${TEMP}
done
# Support storing consul config in subdir eg consul/config/service-defaults/foo.hcl
# Or you can even omit service and use consul/config/defaults/bar.hcl, consul/config/intentions/bar.hcl
for KIND in service-defaults defaults service-intentions intentions service-router router service-resolver resolver; do
if [ -d ${DIR}/consul/config/${KIND} ]; then
for FILE in $(ls ${DIR}/consul/config/${KIND}/*.hcl 2>/dev/null); do
echo "Loading consul conf from ${FILE}"
TEMP=$(mktemp)
replace_conf_var ${FILE} > ${TEMP}
consul config write ${TEMP}
rm -f ${TEMP}
done
for DIR in ./output ./; do
if [ -d "${DIR}/consul/config" ]; then
if [ "$(check_consul_token)" != "1" ]; then
echo "No valid consul token. You have to authenticate first"
kill -INT $$
fi
done
fi
# Note : service-defaults should be loaded before the others
# but that should be the case
for FILE in $(ls ${DIR}/consul/config/*.hcl 2>/dev/null); do
echo "Loading consul conf from ${FILE}"
TEMP=$(mktemp)
replace_conf_var ${FILE} > ${TEMP}
consul config write ${TEMP}
rm -f ${TEMP}
done
# Support storing consul config in subdir eg consul/config/service-defaults/foo.hcl
# Or you can even omit service and use consul/config/defaults/bar.hcl, consul/config/intentions/bar.hcl
for KIND in service-defaults defaults service-intentions intentions service-router router service-resolver resolver; do
if [ -d ${DIR}/consul/config/${KIND} ]; then
for FILE in $(ls ${DIR}/consul/config/${KIND}/*.hcl 2>/dev/null); do
echo "Loading consul conf from ${FILE}"
TEMP=$(mktemp)
replace_conf_var ${FILE} > ${TEMP}
consul config write ${TEMP}
rm -f ${TEMP}
done
fi
done
fi
done
}
# Build all images for the current project
@ -390,30 +384,68 @@ build_image(){
NO_CACHE=""
fi
for DIR in $(find . -name images -type d); do
docker build ${NO_CACHE} -t ${DOCKER_IMAGE} -t ${LATEST} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain $DIR/${IMAGE_NAME} &&
docker push ${DOCKER_IMAGE}
docker push ${LATEST}
break
local FOUND=0
# Try to locate images dir in the current bundles and their dependencies. Use tac to reverse the order
# so dependencies are tried last
for DIR in $(find . -name images -type d | tac); do
if [ -d $DIR/${IMAGE_NAME} ]; then
docker build ${NO_CACHE} -t ${DOCKER_IMAGE} -t ${LATEST} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain $DIR/${IMAGE_NAME} &&
docker push ${DOCKER_IMAGE}
docker push ${LATEST}
FOUND=1
break
fi
done
if [ "${FOUND}" = "0" ]; then
echo "Couldn't find Docker image directory"
kill -INT $$
fi
unset DOCKER_BUILDKIT
}
# Run all executable in the prep.d directory
handle_prep_scripts(){
if [ -d prep.d ]; then
for H in $(find prep.d -type f -o -type l | sort); do
if [ -x "${H}" ]; then
echo "Running script ${H}"
$H $1
for DIR in ./output ./; do
if [ -d "${DIR}/prep.d" ]; then
for H in $(find ${DIR}/prep.d -type f -o -type l | sort); do
if [ -x "${H}" ]; then
echo "Running prep script ${H}"
$H $1
else
echo "Skiping prep script $H (not executable)"
fi
done
fi
done
}
# Update ctctl bundles with git
update_submodules(){
if [ -e ctctl.bundle.conf ]; then
set -o allexport
source ./ctctl.bundle.conf
set +o allexport
for ID in $(printenv | grep -P '^CTCTL_BUNDLE_\d+_NAME=' | sed -E 's/CTCTL_BUNDLE_([0-9]+)_NAME.*/\1/'); do
local NAME=$(printenv CTCTL_BUNDLE_${ID}_NAME)
local URL=$(printenv CTCTL_BUNDLE_${ID}_URL)
local BRANCH=$(printenv CTCTL_BUNDLE_${ID}_BRANCH || echo master)
echo "Working on the ${NAME} bundle"
if [ ! -d bundles/${NAME} ]; then
echo "Adding sub module from ${URL} (branch ${BRANCH})"
git submodule add --branch ${BRANCH} --name ${NAME} --force ${URL} bundles/${NAME}
else
echo "Skiping $H (not executable)"
echo "Updating ${NAME} from ${URL} (branch ${BRANCH})"
git submodule set-branch --branch ${BRANCH} bundles/${NAME}
fi
git submodule update --init --recursive --remote --merge bundles/${NAME}
done
fi
}
# Render templates using levant
# Render templates using gomplate (or levant for backward compat)
render_templates(){
# If a ctctl.bundle.conf file exist, use the new gomplate rendering method
@ -432,14 +464,8 @@ render_templates(){
echo "Working on the ${NAME} bundle"
if [ ! -d bundles/${NAME} ]; then
echo "Adding sub module from ${URL} (branch ${BRANCH})"
git submodule add --branch ${BRANCH} --name ${NAME} --force ${URL} bundles/${NAME}
else
echo "Updating ${NAME} from ${URL} (branch ${BRANCH})"
git submodule set-branch --branch ${BRANCH} bundles/${NAME}
update_submodules
fi
git submodule update --init --recursive --remote --merge bundles/${NAME}
local GOMPLATE_ARGS=(--input-dir "bundles/${NAME}")
# Use [[ and ]] so it won't clash with consul-template fragments
@ -499,6 +525,8 @@ render_templates(){
echo
echo "Formating job files"
find ./ -maxdepth 1 -type f \( -name \*nomad.hcl -o -name \*.nomad \) -exec nomad fmt {} \;
handle_prep_scripts
done
# Cleanup variables
@ -511,6 +539,7 @@ render_templates(){
MERGED_CONF=$(mktemp tmp.XXXXXXXX.yml)
get_merged_conf > ${MERGED_CONF}
# TODO : handle prep scripts with the new bundles system
handle_prep_scripts ${MERGED_CONF}
for TEMPLATE in $(find . -type f -name \*.tpl ! -path "*templates/*"); do
@ -650,6 +679,7 @@ case $1 in
renew_leases
;;
prep)
update_submodules
render_templates
load_policies
load_consul_conf

1
ctctl.example.conf
View File

@ -23,3 +23,4 @@
# LOKI_PASSWORD=S3cr3t.
# Alternatively, you can use a command to fetch the password on demande
# LOKI_PWD_CMD="vault read -field pwd kv/admin/loki"

31
install.sh
View File

@ -31,6 +31,13 @@ case "${SYSTEM}" in
Darwin*) MACHINE=darwin;;
*) MACHINE=unknown;;
esac
# Architesture, so we support Apple M1/M2 chips
CPU=$(uname -m)
case "${ARCH}" in
x86_64) ARCH=amd64;;
arm64) ARCH=arm64;;
*) ARCH=amd64;;
esac
if [ "${MACHINE}" = "unknown" ]; then
echo "Unsupported system"
@ -38,7 +45,7 @@ if [ "${MACHINE}" = "unknown" ]; then
fi
# Check that we have the required utilities
for TOOL in jq openssl curl unzip fzf; do
for TOOL in jq openssl curl unzip fzf git; do
if ! command -v $TOOL > /dev/null 2>&1; then
echo "$TOOL not found. Please install it first"
exit 1
@ -62,9 +69,9 @@ for UTIL in vault consul consul-template nomad levant; do
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
echo "Installing ${UTIL} v${LAST_VER}"
curl -L -O https://releases.hashicorp.com/${UTIL}/${LAST_VER}/${UTIL}_${LAST_VER}_${MACHINE}_amd64.zip
unzip -o -d ~/bin/ ${UTIL}_${LAST_VER}_${MACHINE}_amd64.zip
rm -f ${UTIL}_${LAST_VER}_${MACHINE}_amd64.zip
curl -L -O https://releases.hashicorp.com/${UTIL}/${LAST_VER}/${UTIL}_${LAST_VER}_${MACHINE}_${ARCH}.zip
unzip -o -d ~/bin/ ${UTIL}_${LAST_VER}_${MACHINE}_${ARCH}.zip
rm -f ${UTIL}_${LAST_VER}_${MACHINE}_${ARCH}.zip
else
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
fi
@ -80,10 +87,10 @@ for UTIL in logcli; do
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
echo "Installing ${UTIL} v${LAST_VER}"
curl -L -O https://github.com/grafana/loki/releases/download/v${LAST_VER}/logcli-${MACHINE}-amd64.zip
unzip -o -d ~/bin/ ${UTIL}-${MACHINE}-amd64.zip
rm -f ${UTIL}-${MACHINE}-amd64.zip
mv -f ~/bin/logcli-${MACHINE}-amd64 ~/bin/logcli
curl -L -O https://github.com/grafana/loki/releases/download/v${LAST_VER}/logcli-${MACHINE}-${ARCH}.zip
unzip -o -d ~/bin/ ${UTIL}-${MACHINE}-$ARCH}.zip
rm -f ${UTIL}-${MACHINE}-${ARCH}.zip
mv -f ~/bin/logcli-${MACHINE}-${ARCH} ~/bin/logcli
else
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
fi
@ -99,7 +106,7 @@ for UTIL in yq; do
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
echo "Installing ${UTIL} v${LAST_VER}"
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/mikefarah/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}_amd64
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/mikefarah/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}_${ARCH}
chmod +x ${BIN_DIR}/${UTIL}
else
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
@ -116,7 +123,7 @@ for UTIL in gomplate; do
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
echo "Installing ${UTIL} v${LAST_VER}"
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/hairyhenderson/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}-amd64
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/hairyhenderson/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}-${ARCH}
chmod +x ${BIN_DIR}/${UTIL}
else
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
@ -199,13 +206,13 @@ _EOF
# We'll use consul-template to obtain and renew certificates for Nomad and Consul
for UTIL in nomad consul; do
mkdir -p ~/.ctctl/${INSTALL_DOMAIN}/${UTIL}
# Create the files so realpath do not fail
# Create the files so realpath doesn't fail
# Note : on Linux we could use realpath -m, but not on MAC OS
for FILE in ca.crt cli.crt cli.key cli.p12 bundle.pem; do
touch ~/.ctctl/${INSTALL_DOMAIN}/${UTIL}/${FILE}
done
cat <<_EOF > ~/.ctctl/${INSTALL_DOMAIN}/${UTIL}/bundle.pem.tpl
{{ with pkiCert "pki/${UTIL}/issue/${UTIL}-user" "common_name=${INSTALL_ME}.${UTIL}.${INSTALL_DOMAIN}" "ttl=8760h" }}
{{ with pkiCert "pki/${UTIL}/issue/${UTIL}-user" "common_name=${CTCTL_USER:-$(whoami | sed -r 's/\@.*//')}.${UTIL}.${INSTALL_DOMAIN}" "ttl=8760h" }}
{{ .CA }}
{{ .Cert }}
{{ .Key }}