Add support for gomplate rendering
This commit is contained in:
parent
b38d15e4c4
commit
bee1de9d32
188
ctctl
188
ctctl
|
|
@ -266,25 +266,31 @@ load_policies(){
|
|||
echo "Not currently in a valid env. Run ctctl (with no argument) and select your env first"
|
||||
kill -INT $$
|
||||
fi
|
||||
if [ -d "./vault/policies" ]; then
|
||||
|
||||
local DIR="./"
|
||||
if [ -e ctctl.bundle.conf ]; then
|
||||
DIR="./output"
|
||||
fi
|
||||
|
||||
if [ -d "${DIR}/vault/policies" ]; then
|
||||
if [ "$(check_vault_token)" != "1" ]; then
|
||||
echo "No valid vault token. You have to authenticate first"
|
||||
kill -INT $$
|
||||
fi
|
||||
for PFILE in $(ls ./vault/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)$(get_conf "env_suffix")
|
||||
for PFILE in $(ls ${DIR}/vault/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)
|
||||
echo "Loading vault policy ${PNA}"
|
||||
replace_conf_var ${PFILE} | vault policy write ${PNAME} -
|
||||
done
|
||||
fi
|
||||
if [ -d "./consul/policies" ]; then
|
||||
if [ -d "${DIR}/consul/policies" ]; then
|
||||
if [ "$(check_consul_token)" != "1" ]; then
|
||||
echo "No valid consul token. You have to authenticate first"
|
||||
kill -INT $$
|
||||
fi
|
||||
CONSUL_CUR_POLICIES=$(consul acl policy list -format=json)
|
||||
for PFILE in $(ls ./consul/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)$(get_conf "env_suffix")
|
||||
for PFILE in $(ls ${DIR}/consul/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)
|
||||
# Consul do not use the same command to create a new policy and to update an existing one
|
||||
# so we need to detect if the policy already exists
|
||||
if [ "$(echo ${CONSUL_CUR_POLICIES} | jq -r '.[] | select(.Name=='\"${PNAME}\"') | .Name')" == "${PNAME}" ]; then
|
||||
|
|
@ -296,46 +302,35 @@ load_policies(){
|
|||
fi
|
||||
done
|
||||
fi
|
||||
if [ -d "./nomad/policies" ]; then
|
||||
if [ -d "${DIR}/nomad/policies" ]; then
|
||||
if [ "$(check_nomad_token)" != "1" ]; then
|
||||
echo "No valid nomad token. You have to authenticate first"
|
||||
kill -INT $$
|
||||
fi
|
||||
for PFILE in $(ls ./nomad/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)$(get_conf "env_suffix")
|
||||
for PFILE in $(ls ${DIR}nomad/policies/*.hcl 2>/dev/null); do
|
||||
PNAME=$(basename ${PFILE} .hcl)
|
||||
echo "Loading Nomad policy ${PNAME}"
|
||||
replace_conf_var ${PFILE} | nomad acl policy apply ${PNAME} -
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# Load consul services
|
||||
load_consul_services(){
|
||||
if [ -d "./consul/services" ]; then
|
||||
if [ "$(check_consul_token)" != "1" ]; then
|
||||
echo "No valid consul token. You have to authenticate first"
|
||||
kill -INT $$
|
||||
fi
|
||||
for FILE in $(ls ./consul/services/*.hcl 2>/dev/null); do
|
||||
echo "Registering service from ${FILE}"
|
||||
TEMP=$(mktemp).hcl
|
||||
replace_conf_var ${FILE} > ${TEMP}
|
||||
consul services register ${TEMP}
|
||||
rm -f ${TEMP}
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# Load consul config
|
||||
load_consul_conf(){
|
||||
if [ -d "./consul/config" ]; then
|
||||
|
||||
local DIR="./"
|
||||
if [ -e ctctl.bundle.conf ]; then
|
||||
DIR="./output"
|
||||
fi
|
||||
|
||||
if [ -d "${DIR}/consul/config" ]; then
|
||||
if [ "$(check_consul_token)" != "1" ]; then
|
||||
echo "No valid consul token. You have to authenticate first"
|
||||
kill -INT $$
|
||||
fi
|
||||
# Note : service-defaults should be loaded before the others
|
||||
# but that should be the case
|
||||
for FILE in $(ls ./consul/config/*.hcl 2>/dev/null); do
|
||||
for FILE in $(ls ${DIR}/consul/config/*.hcl 2>/dev/null); do
|
||||
echo "Loading consul conf from ${FILE}"
|
||||
TEMP=$(mktemp)
|
||||
replace_conf_var ${FILE} > ${TEMP}
|
||||
|
|
@ -346,8 +341,8 @@ load_consul_conf(){
|
|||
# Support storing consul config in subdir eg consul/config/service-defaults/foo.hcl
|
||||
# Or you can even omit service and use consul/config/defaults/bar.hcl, consul/config/intentions/bar.hcl
|
||||
for KIND in service-defaults defaults service-intentions intentions service-router router service-resolver resolver; do
|
||||
if [ -d ./consul/config/${KIND} ]; then
|
||||
for FILE in $(ls ./consul/config/${KIND}/*.hcl 2>/dev/null); do
|
||||
if [ -d ${DIR}/consul/config/${KIND} ]; then
|
||||
for FILE in $(ls ${DIR}/consul/config/${KIND}/*.hcl 2>/dev/null); do
|
||||
echo "Loading consul conf from ${FILE}"
|
||||
TEMP=$(mktemp)
|
||||
replace_conf_var ${FILE} > ${TEMP}
|
||||
|
|
@ -386,7 +381,7 @@ build_image(){
|
|||
|
||||
echo "Building image ${DOCKER_IMAGE}"
|
||||
# Extract the basename of the image, removing the repo and the tag
|
||||
local DIR=$(echo -n ${DOCKER_IMAGE} | sed -E 's/.+\/([^\/]+):.*/\1/')
|
||||
local IMAGE_NAME=$(echo -n ${DOCKER_IMAGE} | sed -E 's/.+\/([^\/]+):.*/\1/')
|
||||
export DOCKER_IMAGE=${DOCKER_IMAGE}
|
||||
local LATEST=$(echo ${DOCKER_IMAGE} | sed 's/:.*/:latest/')
|
||||
if [ "${NO_CACHE}" != "" ]; then
|
||||
|
|
@ -394,9 +389,13 @@ build_image(){
|
|||
else
|
||||
NO_CACHE=""
|
||||
fi
|
||||
docker build ${NO_CACHE} -t ${DOCKER_IMAGE} -t ${LATEST} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain images/$DIR &&
|
||||
docker push ${DOCKER_IMAGE}
|
||||
docker push ${LATEST}
|
||||
|
||||
for DIR in $(find . -name images -type d); do
|
||||
docker build ${NO_CACHE} -t ${DOCKER_IMAGE} -t ${LATEST} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain $DIR/${IMAGE_NAME} &&
|
||||
docker push ${DOCKER_IMAGE}
|
||||
docker push ${LATEST}
|
||||
break
|
||||
done
|
||||
unset DOCKER_BUILDKIT
|
||||
}
|
||||
|
||||
|
|
@ -416,22 +415,116 @@ handle_prep_scripts(){
|
|||
|
||||
# Render templates using levant
|
||||
render_templates(){
|
||||
MERGED_CONF=$(mktemp tmp.XXXXXXXX.yml)
|
||||
get_merged_conf > ${MERGED_CONF}
|
||||
|
||||
handle_prep_scripts ${MERGED_CONF}
|
||||
# If a ctctl.bundle.conf file exist, use the new gomplate rendering method
|
||||
if [ -e ctctl.bundle.conf ]; then
|
||||
|
||||
for TEMPLATE in $(find . -type f -name \*.tpl ! -path "*templates/*"); do
|
||||
DIR=$(dirname ${TEMPLATE})
|
||||
FILE=$(basename ${TEMPLATE} .tpl)
|
||||
DEST=${DIR}/${FILE}
|
||||
echo "Rendering ${TEMPLATE} into ${DEST}"
|
||||
# Note: render twice, so included templates get rendered too
|
||||
levant render -var-file ${MERGED_CONF} -log-level=WARN <(levant render -var-file ${MERGED_CONF} -log-level=WARN ${TEMPLATE}) > ${DEST}
|
||||
nomad fmt ${DEST}
|
||||
done
|
||||
set -o allexport
|
||||
source ./ctctl.bundle.conf
|
||||
set +o allexport
|
||||
|
||||
rm -f ${MERGED_CONF}
|
||||
mkdir -p bundles
|
||||
|
||||
for ID in $(printenv | grep -P '^CTCTL_BUNDLE_\d+_NAME=' | sed -E 's/CTCTL_BUNDLE_([0-9]+)_NAME.*/\1/'); do
|
||||
local NAME=$(printenv CTCTL_BUNDLE_${ID}_NAME)
|
||||
local URL=$(printenv CTCTL_BUNDLE_${ID}_URL)
|
||||
local BRANCH=$(printenv CTCTL_BUNDLE_${ID}_BRANCH || echo master)
|
||||
|
||||
echo "Working on the ${NAME} bundle"
|
||||
if [ ! -d bundles/${NAME} ]; then
|
||||
echo "Adding sub module from ${URL} (branch ${BRANCH})"
|
||||
git submodule add --branch ${BRANCH} --name ${NAME} --force ${URL} bundles/${NAME}
|
||||
|
||||
else
|
||||
echo "Updating ${NAME} from ${URL} (branch ${BRANCH})"
|
||||
git submodule set-branch --branch ${BRANCH} bundles/${NAME}
|
||||
fi
|
||||
git submodule update --init --recursive --remote --merge bundles/${NAME}
|
||||
|
||||
local GOMPLATE_ARGS=(--input-dir "bundles/${NAME}")
|
||||
# Use [[ and ]] so it won't clash with consul-template fragments
|
||||
GOMPLATE_ARGS+=(--left-delim '[[' --right-delim ']]')
|
||||
# Do not render templates from dependencies and variables files
|
||||
GOMPLATE_ARGS+=(--exclude .git* --exclude deps/** --exclude variables.yml --exclude images/**)
|
||||
# This is used for two things
|
||||
# - Add the env.suffix to every files (except job files). This permit ctctl to simply infer the policy name from the file name
|
||||
# - Put job files in the current dir for conveniance, and everything else in the output dir
|
||||
GOMPLATE_ARGS+=(--output-map)
|
||||
GOMPLATE_ARGS+=('[[ if (regexp.Match ".*\\.nomad(\\.hcl)?" .in) ]][[ .in ]][[ else ]]output/[[ .in | path.Dir ]]/[[ .in | path.Base | regexp.Replace "^([^\\.]+)\\.(.*)$" (printf "%s%s.%s" "$1" .ctx.env.suffix "$2") ]][[ end ]]')
|
||||
# The context will merge various configuration files to get the variables used to render the templates
|
||||
GOMPLATE_ARGS+=(--context)
|
||||
|
||||
# Build a list of configuration file to merge
|
||||
# Files are in order of precedence (firsts win)
|
||||
I=0
|
||||
local VAR_FILES=""
|
||||
for FILE in ${NOMAD_NAMESPACE}.yml \
|
||||
${NOMAD_NAMESPACE}.yaml \
|
||||
variables.yml \
|
||||
variables.yaml \
|
||||
../${NOMAD_NAMESPACE}.yml \
|
||||
../${NOMAD_NAMESPACE}.yaml \
|
||||
../variables.yml \
|
||||
../variables.yaml \
|
||||
bundles/${NAME}/variables.yml \
|
||||
bundles/${NAME}/variables.yaml \
|
||||
bundles/${NAME}/deps/*/variables.yml \
|
||||
bundles/${NAME}/deps/*/variables.yaml; do
|
||||
if [ -e ${FILE} ]; then
|
||||
if [ $I -eq 0 ]; then
|
||||
VAR_FILES+='.=merge:'
|
||||
else
|
||||
VAR_FILES+='|'
|
||||
fi
|
||||
VAR_FILES+="${FILE}"
|
||||
I=$((I+1))
|
||||
fi
|
||||
done
|
||||
GOMPLATE_ARGS+=("${VAR_FILES}")
|
||||
|
||||
for TEMPLATE_DIR in $(find bundles -type d -name templates); do
|
||||
GOMPLATE_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))=${TEMPLATE_DIR}")
|
||||
# Also declare sub-folders as external templates
|
||||
for SUBFOLDER in $(find ${TEMPLATE_DIR} -mindepth 1 -maxdepth 2 -type d); do
|
||||
GOMPLATE_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))/$(basename ${SUBFOLDER})=${SUBFOLDER}")
|
||||
done
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Redering templates with gomplate ${GOMPLATE_ARGS[@]}"
|
||||
# First, cleanup any previously rendered files
|
||||
rm -rf output ./*.nomad ./*.nomad.hcl
|
||||
gomplate "${GOMPLATE_ARGS[@]}"
|
||||
|
||||
echo
|
||||
echo "Formating job files"
|
||||
find ./ -maxdepth 1 -type f \( -name \*nomad.hcl -o -name \*.nomad \) -exec nomad fmt {} \;
|
||||
done
|
||||
|
||||
# Cleanup variables
|
||||
for VAR in $(printenv | grep -E '^CTCTL_BUNDLE_.*' | sed -E 's/^([^=]+)=.*/\1/'); do
|
||||
unset ${VAR}
|
||||
done
|
||||
|
||||
else
|
||||
# backward compatible, levant based rendering
|
||||
MERGED_CONF=$(mktemp tmp.XXXXXXXX.yml)
|
||||
get_merged_conf > ${MERGED_CONF}
|
||||
|
||||
handle_prep_scripts ${MERGED_CONF}
|
||||
|
||||
for TEMPLATE in $(find . -type f -name \*.tpl ! -path "*templates/*"); do
|
||||
local DIR=$(dirname ${TEMPLATE})
|
||||
local FILE=$(basename ${TEMPLATE} .tpl)
|
||||
local DEST=${DIR}/${FILE}
|
||||
echo "Rendering ${TEMPLATE} into ${DEST}"
|
||||
# Note: render twice, so included templates get rendered too
|
||||
levant render -var-file ${MERGED_CONF} -log-level=WARN <(levant render -var-file ${MERGED_CONF} -log-level=WARN ${TEMPLATE}) > ${DEST}
|
||||
nomad fmt ${DEST}
|
||||
done
|
||||
|
||||
rm -f ${MERGED_CONF}
|
||||
fi
|
||||
}
|
||||
|
||||
# Print Consul and Nomad tokens (not vault, for security reasons)
|
||||
|
|
@ -559,7 +652,6 @@ case $1 in
|
|||
prep)
|
||||
render_templates
|
||||
load_policies
|
||||
load_consul_services
|
||||
load_consul_conf
|
||||
build_required_images
|
||||
renew_leases
|
||||
|
|
|
|||
29
install.sh
29
install.sh
|
|
@ -16,7 +16,7 @@ function get_gh_release() {
|
|||
VERSION=$(cat ~/.ctctl/versions/$1)
|
||||
else
|
||||
VERSION=$(curl --silent "https://api.github.com/repos/$1/releases/latest" | \
|
||||
jq -r .tag_name | perl -pe 's/^v//')
|
||||
jq -r .tag_name | sed -E 's/^v//')
|
||||
# Cache the last version so we don't ask github API every time
|
||||
mkdir -p $(dirname ~/.ctctl/versions/$1)
|
||||
echo -n ${VERSION} > ~/.ctctl/versions/$1
|
||||
|
|
@ -56,7 +56,7 @@ for UTIL in vault consul consul-template nomad levant; do
|
|||
echo "Last version of ${UTIL} is v${LAST_VER}"
|
||||
|
||||
if [ -x "${BIN_DIR}/${UTIL}" ]; then
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version | perl -ne "/^${UTIL} v(\d+(\.\d+)*)/i && print \"\$1\"")
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>/dev/null | head -1 | sed -E 's/.* v([0-9]+(\.[0-9]+)*).*/\1/')
|
||||
echo "${UTIL} v${CUR_VER} is installed"
|
||||
fi
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ done
|
|||
for UTIL in logcli; do
|
||||
LAST_VER=$(get_gh_release "grafana/loki")
|
||||
if [ -x "${BIN_DIR}/${UTIL}" ]; then
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>&1 | perl -ne "/version (\d+(\.\d+)*)/i && print \"\$1\"")
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>&1 | head -1 | sed -E 's/.* version ([0-9]+(\.[0-9]+)*).*/\1/')
|
||||
echo "${UTIL} v${CUR_VER} is installed"
|
||||
fi
|
||||
|
||||
|
|
@ -93,14 +93,31 @@ done
|
|||
for UTIL in yq; do
|
||||
LAST_VER=$(get_gh_release "mikefarah/yq")
|
||||
if [ -x "${BIN_DIR}/${UTIL}" ]; then
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>&1 | perl -ne "/version v(\d+(\.\d+)*)/i && print \"\$1\"")
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>&1 | sed -E 's/.* version v([0-9]+(\.[0-9]+)*).*/\1/')
|
||||
echo "${UTIL} v${CUR_VER} is installed"
|
||||
fi
|
||||
|
||||
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
|
||||
echo "Installing ${UTIL} v${LAST_VER}"
|
||||
curl -L -o ${BIN_DIR}/yq https://github.com/mikefarah/yq/releases/download/v${LAST_VER}/yq_${MACHINE}_amd64
|
||||
chmod +x ${BIN_DIR}/yq
|
||||
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/mikefarah/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}_amd64
|
||||
chmod +x ${BIN_DIR}/${UTIL}
|
||||
else
|
||||
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
|
||||
fi
|
||||
done
|
||||
|
||||
# Install gomplate
|
||||
for UTIL in gomplate; do
|
||||
LAST_VER=$(get_gh_release "hairyhenderson/${UTIL}")
|
||||
if [ -x "${BIN_DIR}/${UTIL}" ]; then
|
||||
CUR_VER=$(${BIN_DIR}/${UTIL} --version 2>&1 | sed -E 's/gomplate version ([0-9]+(\.[0-9]+)*)/\1/')
|
||||
echo "${UTIL} v${CUR_VER} is installed"
|
||||
fi
|
||||
|
||||
if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then
|
||||
echo "Installing ${UTIL} v${LAST_VER}"
|
||||
curl -L -o ${BIN_DIR}/${UTIL} https://github.com/hairyhenderson/${UTIL}/releases/download/v${LAST_VER}/${UTIL}_${MACHINE}-amd64
|
||||
chmod +x ${BIN_DIR}/${UTIL}
|
||||
else
|
||||
echo "Last ${UTIL} version (v${CUR_VER}) is already installed"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in New Issue