Update the way we define bundles and dependencies
Now using a simple bundles.yml file
This commit is contained in:
parent
771b8ab679
commit
95ac66817e
162
ctctl
162
ctctl
|
|
@ -478,28 +478,54 @@ handle_prep_scripts(){
|
|||
done
|
||||
}
|
||||
|
||||
add_submodule(){
|
||||
local NAME=$1
|
||||
local URL=$2
|
||||
local BRANCH=$3
|
||||
local DIR=$4
|
||||
|
||||
mkdir -p ${DIR}
|
||||
if [ ! -d ${DIR}/${NAME} ]; then
|
||||
echo "Adding ${NAME} submodule from ${URL} (branch ${BRANCH})"
|
||||
git submodule add --branch ${BRANCH} --name ${NAME} --force ${URL} ${DIR}/${NAME}
|
||||
else
|
||||
echo "Updating ${NAME} submodule from ${URL} (branch ${BRANCH})"
|
||||
git submodule set-branch --branch ${BRANCH} ${DIR}/${NAME}
|
||||
fi
|
||||
git submodule update --init --recursive --remote --merge ${DIR}/${NAME}
|
||||
}
|
||||
|
||||
# Update ctctl bundles with git
|
||||
update_submodules(){
|
||||
if [ -e ctctl.bundle.conf ]; then
|
||||
set -o allexport
|
||||
source ./ctctl.bundle.conf
|
||||
set +o allexport
|
||||
if [ -e "bundles.yml" ]; then
|
||||
for BUNDLE in $(yq e -o=j -I=0 '.bundles[]' bundles.yml); do
|
||||
local URL=$(echo ${BUNDLE} | jq -r .url)
|
||||
local BRANCH=$(echo ${BUNDLE} | jq -r .branch)
|
||||
local NAME=$(basename ${URL} .git)
|
||||
|
||||
for ID in $(printenv | grep -P '^CTCTL_BUNDLE_\d+_NAME=' | sed -E 's/CTCTL_BUNDLE_([0-9]+)_NAME.*/\1/'); do
|
||||
local NAME=$(printenv CTCTL_BUNDLE_${ID}_NAME)
|
||||
local URL=$(printenv CTCTL_BUNDLE_${ID}_URL)
|
||||
local BRANCH=$(printenv CTCTL_BUNDLE_${ID}_BRANCH || echo master)
|
||||
|
||||
echo "Working on the ${NAME} bundle"
|
||||
if [ ! -d bundles/${NAME} ]; then
|
||||
echo "Adding sub module from ${URL} (branch ${BRANCH})"
|
||||
git submodule add --branch ${BRANCH} --name ${NAME} --force ${URL} bundles/${NAME}
|
||||
|
||||
else
|
||||
echo "Updating ${NAME} from ${URL} (branch ${BRANCH})"
|
||||
git submodule set-branch --branch ${BRANCH} bundles/${NAME}
|
||||
# If the branch is not defined, default to master
|
||||
if [ "${BRANCH}" = "null" ]; then
|
||||
BRANCH=master
|
||||
fi
|
||||
add_submodule ${NAME} ${URL} ${BRANCH} bundles
|
||||
if [ -e "bundles/${NAME}/bundles.yml" ]; then
|
||||
for DEP in $(yq e -o=j -I=0 '.dependencies[]' bundles/${NAME}/bundles.yml); do
|
||||
local DEP_URL=$(echo ${DEP} | jq -r .url)
|
||||
local DEP_BRANCH=$(echo ${DEP} | jq -r .branch)
|
||||
local DEP_NAME="${NAME}_$(basename ${DEP_URL} .git)"
|
||||
|
||||
# If the branch is not defined, assume the same as the parent bundle
|
||||
if [ "${DEP_BRANCH}" = "null" ]; then
|
||||
DEP_BRANCH=${BRANCH}
|
||||
fi
|
||||
|
||||
# Handle relative URL for dependencies
|
||||
if echo ${DEP_URL} | grep -qE '^\.\./'; then
|
||||
DEP_URL=$(dirname ${URL})/$(echo ${DEP_URL} | sed -E 's|^\.\./||')
|
||||
fi
|
||||
add_submodule ${DEP_NAME} ${DEP_URL} ${DEP_BRANCH} bundles
|
||||
done
|
||||
fi
|
||||
git submodule update --init --recursive --remote --merge bundles/${NAME}
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
|
@ -507,19 +533,20 @@ update_submodules(){
|
|||
# Render templates using gomplate (or levant for backward compat)
|
||||
render_templates(){
|
||||
|
||||
# If a ctctl.bundle.conf file exist, use the new gomplate rendering method
|
||||
if [ -e ctctl.bundle.conf ]; then
|
||||
|
||||
set -o allexport
|
||||
source ./ctctl.bundle.conf
|
||||
set +o allexport
|
||||
|
||||
# If a bundles.yml file exist, use the new gomplate rendering method
|
||||
if [ -e bundles.yml ]; then
|
||||
mkdir -p bundles
|
||||
|
||||
for ID in $(printenv | grep -P '^CTCTL_BUNDLE_\d+_NAME=' | sed -E 's/CTCTL_BUNDLE_([0-9]+)_NAME.*/\1/'); do
|
||||
local NAME=$(printenv CTCTL_BUNDLE_${ID}_NAME)
|
||||
local URL=$(printenv CTCTL_BUNDLE_${ID}_URL)
|
||||
local BRANCH=$(printenv CTCTL_BUNDLE_${ID}_BRANCH || echo master)
|
||||
CONFIG=$(mktemp -t tmp.XXXXXXXXX.yml)
|
||||
|
||||
for BUNDLE in $(yq e -o=j -I=0 '.bundles[]' bundles.yml); do
|
||||
local URL=$(echo ${BUNDLE} | jq -r .url)
|
||||
local BRANCH=$(echo ${BUNDLE} | jq -r .branch)
|
||||
local NAME=$(basename ${URL} .git)
|
||||
|
||||
if [ "${BRANCH}" = "null" ]; then
|
||||
BRANCH=master
|
||||
fi
|
||||
|
||||
echo "Working on the ${NAME} bundle"
|
||||
if [ ! -d bundles/${NAME} ]; then
|
||||
|
|
@ -530,8 +557,39 @@ render_templates(){
|
|||
local GOMPLATE_COMMON_ARGS=(--left-delim '[[' --right-delim ']]')
|
||||
# Setup a vault datasource
|
||||
GOMPLATE_COMMON_ARGS+=(-d vault=vault:///)
|
||||
# The context will merge various configuration files to get the variables used to render the templates
|
||||
GOMPLATE_COMMON_ARGS+=(--context)
|
||||
|
||||
# Declare named, external templates
|
||||
if [ -d "bundles/${NAME}/templates" ]; then
|
||||
GOMPLATE_COMMON_ARGS+=(--template ${NAME}=bundles/${NAME}/templates)
|
||||
for SUBFOLDER in $(find bundles/${NAME}/templates -mindepth 1 -maxdepth 2 -type d); do
|
||||
GOMPLATE_COMMON_ARGS+=(--template ${NAME}/$(basename ${SUBFOLDER})=${SUBFOLDER})
|
||||
done
|
||||
fi
|
||||
if [ -e "bundles/${NAME}/bundles.yml" ]; then
|
||||
for DEP in $(yq e -o=j -I=0 '.dependencies[]' bundles/${NAME}/bundles.yml); do
|
||||
local DEP_URL=$(echo $DEP | jq -r .url)
|
||||
local DEP_NAME="$(basename ${DEP_URL} .git)"
|
||||
if [ -d "bundles/${NAME}_${DEP_NAME}/templates" ]; then
|
||||
GOMPLATE_COMMON_ARGS+=(--template ${DEP_NAME}=bundles/${NAME}_${DEP_NAME}/templates)
|
||||
for SUBFOLDER in $(find bundles/${NAME}_${DEP_NAME}/templates -mindepth 1 -maxdepth 2 -type d); do
|
||||
GOMPLATE_COMMON_ARGS+=(--template ${DEP_NAME}/$(basename ${SUBFOLDER})=${SUBFOLDER})
|
||||
done
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
local GOMPLATE_BUNDLE_ARGS=(--input-dir "bundles/${NAME}")
|
||||
# Do not render templates from dependencies, variables files and images (images will be handled later)
|
||||
GOMPLATE_BUNDLE_ARGS+=(--exclude .git* --exclude deps/** --exclude variables.yml --exclude images/** --exclude templates/**)
|
||||
# This is used for two things
|
||||
# - Add the env.suffix to every files (except job files). This permit ctctl to simply infer the policy name from the file name
|
||||
# - Put job files in the current dir for conveniance, and everything else in the output dir
|
||||
GOMPLATE_BUNDLE_ARGS+=(--output-map)
|
||||
GOMPLATE_BUNDLE_ARGS+=('[[ if (regexp.Match ".*\\.nomad(\\.hcl)?" .in) ]][[ .in ]][[ else ]]output/[[ .in | path.Dir ]]/[[ .in | path.Base | regexp.Replace "^([^\\.]+)\\.(.*)$" (printf "%s%s.%s" "$1" .ctx.env.suffix "$2") ]][[ end ]]')
|
||||
|
||||
echo
|
||||
|
||||
# Now, render the merged config in a temp file
|
||||
|
||||
# Build a list of configuration file to merge
|
||||
# Files are in order of precedence (firsts win)
|
||||
|
|
@ -547,8 +605,8 @@ render_templates(){
|
|||
../variables.yaml \
|
||||
bundles/${NAME}/variables.yml \
|
||||
bundles/${NAME}/variables.yaml \
|
||||
bundles/${NAME}/deps/*/variables.yml \
|
||||
bundles/${NAME}/deps/*/variables.yaml; do
|
||||
bundles/*/variables.yml \
|
||||
bundles/*/variables.yaml; do
|
||||
if [ -e ${FILE} ]; then
|
||||
if [ $I -eq 0 ]; then
|
||||
VAR_FILES+='.=merge:'
|
||||
|
|
@ -559,35 +617,18 @@ render_templates(){
|
|||
I=$((I+1))
|
||||
fi
|
||||
done
|
||||
GOMPLATE_COMMON_ARGS+=("${VAR_FILES}")
|
||||
gomplate "${GOMPLATE_COMMON_ARGS[@]}" --context "${VAR_FILES[@]}" -i "[[ . | toYAML ]]" > ${CONFIG}
|
||||
# And render it again so we can replace any templated values in the config itself
|
||||
gomplate "${GOMPLATE_COMMON_ARGS[@]}" --context ${VAR_FILES} -f ${CONFIG} -o ${CONFIG}
|
||||
GOMPLATE_COMMON_ARGS+=(--context .=file://${CONFIG})
|
||||
|
||||
for TEMPLATE_DIR in $(find bundles -type d -name templates); do
|
||||
GOMPLATE_COMMON_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))=${TEMPLATE_DIR}")
|
||||
# Also declare sub-folders as external templates
|
||||
for SUBFOLDER in $(find ${TEMPLATE_DIR} -mindepth 1 -maxdepth 2 -type d); do
|
||||
GOMPLATE_COMMON_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))/$(basename ${SUBFOLDER})=${SUBFOLDER}")
|
||||
done
|
||||
done
|
||||
|
||||
local GOMPLATE_BUNDLE_ARGS=(--input-dir "bundles/${NAME}")
|
||||
# Do not render templates from dependencies, variables files and images (images will be handled later)
|
||||
GOMPLATE_BUNDLE_ARGS+=(--exclude .git* --exclude deps/** --exclude variables.yml --exclude images/** --exclude templates/**)
|
||||
# This is used for two things
|
||||
# - Add the env.suffix to every files (except job files). This permit ctctl to simply infer the policy name from the file name
|
||||
# - Put job files in the current dir for conveniance, and everything else in the output dir
|
||||
GOMPLATE_BUNDLE_ARGS+=(--output-map)
|
||||
GOMPLATE_BUNDLE_ARGS+=('[[ if (regexp.Match ".*\\.nomad(\\.hcl)?" .in) ]][[ .in ]][[ else ]]output/[[ .in | path.Dir ]]/[[ .in | path.Base | regexp.Replace "^([^\\.]+)\\.(.*)$" (printf "%s%s.%s" "$1" .ctx.env.suffix "$2") ]][[ end ]]')
|
||||
|
||||
echo
|
||||
echo "Redering bundles with gomplate ${GOMPLATE_COMMON_ARGS[@]} ${GOMPLATE_BUNDLE_ARGS[@]}"
|
||||
|
||||
# First, cleanup any previously rendered files
|
||||
rm -rf output ./*.nomad ./*.nomad.hcl
|
||||
|
||||
# Now render the bundle files
|
||||
gomplate "${GOMPLATE_COMMON_ARGS[@]}" "${GOMPLATE_BUNDLE_ARGS[@]}"
|
||||
# Render a second time job files. This is a bit hacky, but allows
|
||||
# configuration to refer to other configuration key
|
||||
for JOB in ./*.nomad.hcl; do
|
||||
grep -q '[[' ${JOB} && gomplate "${GOMPLATE_COMMON_ARGS[@]}" -f ${JOB} -o ${JOB}
|
||||
done
|
||||
|
||||
for IMGDIR in $(find . -name images -type d); do
|
||||
for DOCKER_IMAGE in $(find ${IMGDIR} -mindepth 1 -maxdepth 1 -type d); do
|
||||
|
|
@ -607,13 +648,10 @@ render_templates(){
|
|||
|
||||
# Run prep.d scripts
|
||||
handle_render_scripts
|
||||
done
|
||||
|
||||
# Cleanup variables
|
||||
for VAR in $(printenv | grep -E '^CTCTL_BUNDLE_.*' | sed -E 's/^([^=]+)=.*/\1/'); do
|
||||
unset ${VAR}
|
||||
# And now relete the config
|
||||
#rm -f ${CONFIG}
|
||||
done
|
||||
|
||||
else
|
||||
# backward compatible, levant based rendering
|
||||
MERGED_CONF=$(mktemp tmp.XXXXXXXX.yml)
|
||||
|
|
|
|||
Loading…
Reference in New Issue