From 9aa66a32afd63b474254bb18b94c65d82a18916f Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Tue, 29 Aug 2023 23:55:28 +0200 Subject: [PATCH] Render images, exclude templates direct rendering & various enhancements --- ctctl | 58 ++++++++++++++++++++++++++++++++---------------------- install.sh | 2 +- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/ctctl b/ctctl index 5c08428..8482244 100755 --- a/ctctl +++ b/ctctl @@ -333,7 +333,7 @@ load_consul_conf(){ # Support storing consul config in subdir eg consul/config/service-defaults/foo.hcl # Or you can even omit service and use consul/config/defaults/bar.hcl, consul/config/intentions/bar.hcl - for KIND in service-defaults defaults service-intentions intentions service-router router service-resolver resolver; do + for KIND in service-defaults service-intentions service-router service-resolver proxy-defaults; do if [ -d ${DIR}/consul/config/${KIND} ]; then for FILE in $(ls ${DIR}/consul/config/${KIND}/*.hcl 2>/dev/null); do echo "Loading consul conf from ${FILE}" @@ -385,12 +385,10 @@ build_image(){ fi local FOUND=0 - # Try to locate images dir in the current bundles and their dependencies. Use tac to reverse the order - # so dependencies are tried last - for DIR in $(find . -name images -type d | tac); do + for DIR in ./images ./output/images; do if [ -d $DIR/${IMAGE_NAME} ]; then - docker build ${NO_CACHE} -t ${DOCKER_IMAGE} -t ${LATEST} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain $DIR/${IMAGE_NAME} && - docker push ${DOCKER_IMAGE} + docker build ${NO_CACHE} -t ${DOCKER_IMAGE} ${CTCTL_DOCKER_BUILD_OPTS:-} --progress=plain $DIR/${IMAGE_NAME} &&\ + docker push ${DOCKER_IMAGE} &&\ docker push ${LATEST} FOUND=1 break @@ -467,18 +465,10 @@ render_templates(){ update_submodules fi - local GOMPLATE_ARGS=(--input-dir "bundles/${NAME}") # Use [[ and ]] so it won't clash with consul-template fragments - GOMPLATE_ARGS+=(--left-delim '[[' --right-delim ']]') - # Do not render templates from dependencies and variables files - GOMPLATE_ARGS+=(--exclude .git* --exclude deps/** --exclude variables.yml --exclude images/**) - # This is used for two things - # - Add the env.suffix to every files (except job files). This permit ctctl to simply infer the policy name from the file name - # - Put job files in the current dir for conveniance, and everything else in the output dir - GOMPLATE_ARGS+=(--output-map) - GOMPLATE_ARGS+=('[[ if (regexp.Match ".*\\.nomad(\\.hcl)?" .in) ]][[ .in ]][[ else ]]output/[[ .in | path.Dir ]]/[[ .in | path.Base | regexp.Replace "^([^\\.]+)\\.(.*)$" (printf "%s%s.%s" "$1" .ctx.env.suffix "$2") ]][[ end ]]') + local GOMPLATE_COMMON_ARGS=(--left-delim '[[' --right-delim ']]') # The context will merge various configuration files to get the variables used to render the templates - GOMPLATE_ARGS+=(--context) + GOMPLATE_COMMON_ARGS+=(--context) # Build a list of configuration file to merge # Files are in order of precedence (firsts win) @@ -506,26 +496,48 @@ render_templates(){ I=$((I+1)) fi done - GOMPLATE_ARGS+=("${VAR_FILES}") + GOMPLATE_COMMON_ARGS+=("${VAR_FILES}") for TEMPLATE_DIR in $(find bundles -type d -name templates); do - GOMPLATE_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))=${TEMPLATE_DIR}") + GOMPLATE_COMMON_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))=${TEMPLATE_DIR}") # Also declare sub-folders as external templates for SUBFOLDER in $(find ${TEMPLATE_DIR} -mindepth 1 -maxdepth 2 -type d); do - GOMPLATE_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))/$(basename ${SUBFOLDER})=${SUBFOLDER}") + GOMPLATE_COMMON_ARGS+=(--template "$(basename $(dirname ${TEMPLATE_DIR}))/$(basename ${SUBFOLDER})=${SUBFOLDER}") done done + local GOMPLATE_BUNDLE_ARGS=(--input-dir "bundles/${NAME}") + # Do not render templates from dependencies, variables files and images (images will be handled later) + GOMPLATE_BUNDLE_ARGS+=(--exclude .git* --exclude deps/** --exclude variables.yml --exclude images/** --exclude templates/**) + # This is used for two things + # - Add the env.suffix to every files (except job files). This permit ctctl to simply infer the policy name from the file name + # - Put job files in the current dir for conveniance, and everything else in the output dir + GOMPLATE_BUNDLE_ARGS+=(--output-map) + GOMPLATE_BUNDLE_ARGS+=('[[ if (regexp.Match ".*\\.nomad(\\.hcl)?" .in) ]][[ .in ]][[ else ]]output/[[ .in | path.Dir ]]/[[ .in | path.Base | regexp.Replace "^([^\\.]+)\\.(.*)$" (printf "%s%s.%s" "$1" .ctx.env.suffix "$2") ]][[ end ]]') + echo - echo "Redering templates with gomplate ${GOMPLATE_ARGS[@]}" + echo "Redering bundles with gomplate ${GOMPLATE_COMMON_ARGS[@]} ${GOMPLATE_BUNDLE_ARGS[@]}" # First, cleanup any previously rendered files rm -rf output ./*.nomad ./*.nomad.hcl - gomplate "${GOMPLATE_ARGS[@]}" + gomplate "${GOMPLATE_COMMON_ARGS[@]}" "${GOMPLATE_BUNDLE_ARGS[@]}" + + for IMGDIR in $(find . -name images -type d); do + for DOCKER_IMAGE in $(find ${IMGDIR} -mindepth 1 -maxdepth 1 -type d); do + echo "Redering Docker image $(basename ${DOCKER_IMAGE})" + gomplate "${GOMPLATE_COMMON_ARGS[@]}" --input-dir ${DOCKER_IMAGE} --exclude resources/** --exclude root/** --output-dir output/images/$(basename ${DOCKER_IMAGE})/ + for ROOT in resources root; do + if [ -d "${DOCKER_IMAGE}/${ROOT}" ]; then + cp -r "${DOCKER_IMAGE}/${ROOT}" output/images/$(basename ${DOCKER_IMAGE})/ + fi + done + done + done echo echo "Formating job files" find ./ -maxdepth 1 -type f \( -name \*nomad.hcl -o -name \*.nomad \) -exec nomad fmt {} \; + # Run prep.d scripts handle_prep_scripts done @@ -648,8 +660,6 @@ get_conf(){ get_merged_conf | yq ".$1" } -FZF_CMD=${CTCTL_FZF_CMD:-fzf --height=~10% --cycle --bind 'space:toggle' --marker='*'} - # Return a space separated list of jobs the current dir get_job_list(){ local JOBS="" @@ -659,6 +669,8 @@ get_job_list(){ echo $JOBS } +FZF_CMD=${CTCTL_FZF_CMD:-fzf --height=~10% --cycle --bind 'space:toggle' --marker='*'} + case $1 in current) current_env diff --git a/install.sh b/install.sh index 0f95796..c320f9f 100755 --- a/install.sh +++ b/install.sh @@ -88,7 +88,7 @@ for UTIL in logcli; do if [ ! -x "${BIN_DIR}/${UTIL}" ] || ! echo -e "${LAST_VER}\n${CUR_VER}" | sort --version-sort --check > /dev/null 2>&1; then echo "Installing ${UTIL} v${LAST_VER}" curl -L -O https://github.com/grafana/loki/releases/download/v${LAST_VER}/logcli-${MACHINE}-${ARCH}.zip - unzip -o -d ~/bin/ ${UTIL}-${MACHINE}-$ARCH}.zip + unzip -o -d ~/bin/ ${UTIL}-${MACHINE}-${ARCH}.zip rm -f ${UTIL}-${MACHINE}-${ARCH}.zip mv -f ~/bin/logcli-${MACHINE}-${ARCH} ~/bin/logcli else