From 653dcb409126e4743f2cc49baa489f41cebc74db Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Fri, 15 Jul 2022 21:40:15 -0700 Subject: [PATCH 01/83] chore: update alpine image. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0685c21..a5dfb47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.12 as rq-build +FROM alpine:3.16 as rq-build ENV RQ_VERSION=1.0.2 WORKDIR /root/ From 9701554b3ca7d5c5d7bcf429dd375dc98245a5a2 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:13:11 -0700 Subject: [PATCH 02/83] fix: SC2086 shellcheck fixes. --- docker-entrypoint | 96 +++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 2696ee5..ef27cf9 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -17,13 +17,13 @@ fi get_config() { if [ -f "${HOME_DIR}/config.json" ]; then - jq 'map(.)' ${HOME_DIR}/config.json > ${HOME_DIR}/config.working.json + jq 'map(.)' "${HOME_DIR}"/config.json > "${HOME_DIR}"/config.working.json elif [ -f "${HOME_DIR}/config.toml" ]; then - rq -t <<< $(cat ${HOME_DIR}/config.toml) | jq 'map(.)' > ${HOME_DIR}/config.json + rq -t <<< $(cat "${HOME_DIR}"/config.toml) | jq 'map(.)' > "${HOME_DIR}"/config.json elif [ -f "${HOME_DIR}/config.yml" ]; then - rq -y <<< $(cat ${HOME_DIR}/config.yml) | jq 'map(.)' > ${HOME_DIR}/config.json + rq -y <<< $(cat "${HOME_DIR}"/config.yml) | jq 'map(.)' > "${HOME_DIR}"/config.json elif [ -f "${HOME_DIR}/config.yaml" ]; then - rq -y <<< $(cat ${HOME_DIR}/config.yaml) | jq 'map(.)' > ${HOME_DIR}/config.json + rq -y <<< $(cat "${HOME_DIR}"/config.yaml) | jq 'map(.)' > "${HOME_DIR}"/config.json fi } @@ -31,7 +31,7 @@ DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker # Ensure dir exist - in case of volume mapping -mkdir -p ${HOME_DIR}/jobs ${HOME_DIR}/projects +mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects ensure_docker_socket_accessible() { if ! grep -q "^docker:" /etc/group; then @@ -40,12 +40,12 @@ ensure_docker_socket_accessible() { if [ "${DOCKER_GID}" != "0" ]; then if ! grep -qE "^[^:]+:[^:]+:${DOCKER_GID}:" /etc/group; then # No group with such gid exists - create group docker - addgroup -g ${DOCKER_GID} docker + addgroup -g "${DOCKER_GID}" docker adduser docker docker else # Group with such gid exists - add user "docker" to this group DOCKER_GROUP_NAME=`getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}'` - adduser docker $DOCKER_GROUP_NAME + adduser docker "$DOCKER_GROUP_NAME" fi else # Docker socket belongs to "root" group - add user "docker" to this group @@ -59,14 +59,14 @@ slugify() { } make_image_cmd() { - DOCKERARGS=$(echo ${1} | jq -r .dockerargs) - VOLUMES=$(echo ${1} | jq -r '.volumes | map(" -v " + .) | join("")') - PORTS=$(echo ${1} | jq -r '.ports | map(" -p " + .) | join("")') - EXPOSE=$(echo ${1} | jq -r '.expose | map(" --expose " + .) | join("")') + DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) + VOLUMES=$(echo "${1}" | jq -r '.volumes | map(" -v " + .) | join("")') + PORTS=$(echo "${1}" | jq -r '.ports | map(" -p " + .) | join("")') + EXPOSE=$(echo "${1}" | jq -r '.expose | map(" --expose " + .) | join("")') # We'll add name in, if it exists - NAME=$(echo ${1} | jq -r 'select(.name != null) | .name') - NETWORK=$(echo ${1} | jq -r 'select(.network != null) | .network') - ENVIRONMENT=$(echo ${1} | jq -r '.environment | map(" -e " + .) | join("")') + NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') + NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network') + ENVIRONMENT=$(echo "${1}" | jq -r '.environment | map(" -e " + .) | join("")') # echo ${1} | jq -r '.environment | join("\n")' > ${PWD}/${NAME}.env # ENVIRONMENT=" --env-file ${PWD}/${NAME}.env" if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi @@ -76,19 +76,19 @@ make_image_cmd() { if [ ! -z "${ENVIRONMENT}" ]; then DOCKERARGS="${DOCKERARGS}${ENVIRONMENT}"; fi if [ ! -z "${PORTS}" ]; then DOCKERARGS="${DOCKERARGS}${PORTS}"; fi if [ ! -z "${EXPOSE}" ]; then DOCKERARGS="${DOCKERARGS}${EXPOSE}"; fi - IMAGE=$(echo ${1} | jq -r .image | envsubst) - TMP_COMMAND=$(echo ${1} | jq -r .command) + IMAGE=$(echo "${1}" | jq -r .image | envsubst) + TMP_COMMAND=$(echo "${1}" | jq -r .command) echo "docker run ${DOCKERARGS} ${IMAGE} ${TMP_COMMAND}" } make_container_cmd() { - DOCKERARGS=$(echo ${1} | jq -r .dockerargs) + DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi - SCRIPT_NAME=$(echo ${1} | jq -r .name) - SCRIPT_NAME=$(slugify $SCRIPT_NAME) - PROJECT=$(echo ${1} | jq -r .project) - CONTAINER=$(echo ${1} | jq -r .container | envsubst) - TMP_COMMAND=$(echo ${1} | jq -r .command) + SCRIPT_NAME=$(echo "${1}" | jq -r .name) + SCRIPT_NAME=$(slugify "$SCRIPT_NAME") + PROJECT=$(echo "${1}" | jq -r .project) + CONTAINER=$(echo "${1}" | jq -r .container | envsubst) + TMP_COMMAND=$(echo "${1}" | jq -r .command) if [ "${PROJECT}" != "null" ]; then @@ -96,7 +96,7 @@ make_container_cmd() { if [ "${SCRIPT_NAME}" == "null" ]; then SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) fi -cat << EOF > ${HOME_DIR}/projects/${SCRIPT_NAME}.sh +cat << EOF > "${HOME_DIR}"/projects/"${SCRIPT_NAME}".sh #!/usr/bin/env bash set -e @@ -119,14 +119,14 @@ EOF #} make_cmd() { - if [ "$(echo ${1} | jq -r .image)" != "null" ]; then + if [ "$(echo "${1}" | jq -r .image)" != "null" ]; then make_image_cmd "$1" - elif [ "$(echo ${1} | jq -r .container)" != "null" ]; then + elif [ "$(echo "${1}" | jq -r .container)" != "null" ]; then make_container_cmd "$1" #elif [ "$(echo ${1} | jq -r .host)" != "null" ]; then # make_host_cmd "$1" else - echo ${1} | jq -r .command + echo "${1}" | jq -r .command fi } @@ -157,9 +157,9 @@ parse_schedule() { TIME=$2 TOTAL=0 - M=$(echo $TIME | grep -o '[0-9]\+m') - H=$(echo $TIME | grep -o '[0-9]\+h') - D=$(echo $TIME | grep -o '[0-9]\+d') + M=$(echo "$TIME" | grep -o '[0-9]\+m') + H=$(echo "$TIME" | grep -o '[0-9]\+h') + D=$(echo "$TIME" | grep -o '[0-9]\+d') if [ -n "${M}" ]; then TOTAL=$(($TOTAL + ${M::-1})) @@ -186,31 +186,31 @@ function build_crontab() { ONSTART=() while read i ; do - SCHEDULE=$(jq -r .[$i].schedule ${CONFIG} | sed 's/\*/\\*/g') + SCHEDULE=$(jq -r .["$i"].schedule "${CONFIG}" | sed 's/\*/\\*/g') if [ "${SCHEDULE}" == "null" ]; then - echo "Schedule Missing: $(jq -r .[$i].schedule ${CONFIG})" + echo "Schedule Missing: $(jq -r .["$i"].schedule "${CONFIG}")" continue fi - SCHEDULE=$(parse_schedule ${SCHEDULE} | sed 's/\\//g') + SCHEDULE=$(parse_schedule "${SCHEDULE}" | sed 's/\\//g') - if [ "$(jq -r .[$i].command ${CONFIG})" == "null" ]; then - echo "Command Missing: $(jq -r .[$i].command ${CONFIG})" + if [ "$(jq -r .["$i"].command "${CONFIG}")" == "null" ]; then + echo "Command Missing: $(jq -r .["$i"].command "${CONFIG}")" continue fi - COMMENT=$(jq -r .[$i].comment ${CONFIG}) + COMMENT=$(jq -r .["$i"].comment "${CONFIG}") if [ "${COMMENT}" != "null" ]; then echo "# ${COMMENT}" >> ${CRONTAB_FILE} fi - SCRIPT_NAME=$(jq -r .[$i].name ${CONFIG}) - SCRIPT_NAME=$(slugify $SCRIPT_NAME) + SCRIPT_NAME=$(jq -r .["$i"].name "${CONFIG}") + SCRIPT_NAME=$(slugify "$SCRIPT_NAME") if [ "${SCRIPT_NAME}" == "null" ]; then SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) fi COMMAND="/bin/bash ${HOME_DIR}/jobs/${SCRIPT_NAME}.sh" -cat << EOF > ${HOME_DIR}/jobs/${SCRIPT_NAME}.sh +cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #!/usr/bin/env bash set -e @@ -222,31 +222,31 @@ set -e echo "Start Cronjob **${SCRIPT_NAME}** ${COMMENT}" -$(make_cmd "$(jq -c .[$i] ${CONFIG})") +$(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF - if [ "$(jq -r .[$i].trigger ${CONFIG})" != "null" ]; then + if [ "$(jq -r .["$i"].trigger "${CONFIG}")" != "null" ]; then while read j ; do - if [ "$(jq .[$i].trigger[$j].command ${CONFIG})" == "null" ]; then - echo "Command Missing: $(jq -r .[$i].trigger[$j].command ${CONFIG})" + if [ "$(jq .["$i"].trigger["$j"].command "${CONFIG}")" == "null" ]; then + echo "Command Missing: $(jq -r .["$i"].trigger["$j"].command "${CONFIG}")" continue fi #TRIGGER_COMMAND=$(make_cmd "$(jq -c .[$i].trigger[$j] ${CONFIG})") - echo "$(make_cmd "$(jq -c .[$i].trigger[$j] ${CONFIG})")" >> ${HOME_DIR}/jobs/${SCRIPT_NAME}.sh + echo "$(make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #COMMAND="${COMMAND} && ${TRIGGER_COMMAND}" - done < <(jq -r '.['$i'].trigger|keys[]' ${CONFIG}) + done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi - echo "echo \"End Cronjob **${SCRIPT_NAME}** ${COMMENT}\"" >> ${HOME_DIR}/jobs/${SCRIPT_NAME}.sh + echo "echo \"End Cronjob **${SCRIPT_NAME}** ${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh echo "${SCHEDULE} ${COMMAND}" >> ${CRONTAB_FILE} - if [ "$(jq -r .[$i].onstart ${CONFIG})" == "true" ]; then + if [ "$(jq -r .["$i"].onstart "${CONFIG}")" == "true" ]; then ONSTART+=("${COMMAND}") fi - done < <(jq -r '.|keys[]' ${CONFIG}) + done < <(jq -r '.|keys[]' "${CONFIG}") echo "##### crontab generation complete #####" cat ${CRONTAB_FILE} @@ -271,7 +271,7 @@ start_app() { echo "NO CONFIG FILE FOUND" fi if [ "$1" = "crond" ]; then - if [ -f ${CONFIG} ]; then + if [ -f "${CONFIG}" ]; then build_crontab else echo "Unable to find ${CONFIG}" From 82875aba6e09ab8a0d777e7aa2aec0910beccef4 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:14:30 -0700 Subject: [PATCH 03/83] fix: SC2004 shellcheck fixes. --- docker-entrypoint | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index ef27cf9..e3caa03 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -162,13 +162,13 @@ parse_schedule() { D=$(echo "$TIME" | grep -o '[0-9]\+d') if [ -n "${M}" ]; then - TOTAL=$(($TOTAL + ${M::-1})) + TOTAL=$((TOTAL + ${M::-1})) fi if [ -n "${H}" ]; then - TOTAL=$(($TOTAL + ${H::-1} * 60)) + TOTAL=$((TOTAL + ${H::-1} * 60)) fi if [ -n "${D}" ]; then - TOTAL=$(($TOTAL + ${D::-1} * 60 * 24)) + TOTAL=$((TOTAL + ${D::-1} * 60 * 24)) fi echo "*/${TOTAL} * * * *" From 8c8e19c1c982330c7ef9b687629ff294c87a6ac9 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:15:18 -0700 Subject: [PATCH 04/83] fix: SC2006 shellcheck fixes. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index e3caa03..987e178 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -44,7 +44,7 @@ ensure_docker_socket_accessible() { adduser docker docker else # Group with such gid exists - add user "docker" to this group - DOCKER_GROUP_NAME=`getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}'` + DOCKER_GROUP_NAME=$(getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}') adduser docker "$DOCKER_GROUP_NAME" fi else From 4d945526fe74057a8cf4f8173be8b42917c55a4d Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:16:22 -0700 Subject: [PATCH 05/83] fix: SC2236 shellcheck fixes. --- docker-entrypoint | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 987e178..3d3eaba 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -70,12 +70,12 @@ make_image_cmd() { # echo ${1} | jq -r '.environment | join("\n")' > ${PWD}/${NAME}.env # ENVIRONMENT=" --env-file ${PWD}/${NAME}.env" if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi - if [ ! -z "${NAME}" ]; then DOCKERARGS="${DOCKERARGS} --rm --name ${NAME} "; fi - if [ ! -z "${NETWORK}" ]; then DOCKERARGS="${DOCKERARGS} --network ${NETWORK} "; fi - if [ ! -z "${VOLUMES}" ]; then DOCKERARGS="${DOCKERARGS}${VOLUMES}"; fi - if [ ! -z "${ENVIRONMENT}" ]; then DOCKERARGS="${DOCKERARGS}${ENVIRONMENT}"; fi - if [ ! -z "${PORTS}" ]; then DOCKERARGS="${DOCKERARGS}${PORTS}"; fi - if [ ! -z "${EXPOSE}" ]; then DOCKERARGS="${DOCKERARGS}${EXPOSE}"; fi + if [ -n "${NAME}" ]; then DOCKERARGS="${DOCKERARGS} --rm --name ${NAME} "; fi + if [ -n "${NETWORK}" ]; then DOCKERARGS="${DOCKERARGS} --network ${NETWORK} "; fi + if [ -n "${VOLUMES}" ]; then DOCKERARGS="${DOCKERARGS}${VOLUMES}"; fi + if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS="${DOCKERARGS}${ENVIRONMENT}"; fi + if [ -n "${PORTS}" ]; then DOCKERARGS="${DOCKERARGS}${PORTS}"; fi + if [ -n "${EXPOSE}" ]; then DOCKERARGS="${DOCKERARGS}${EXPOSE}"; fi IMAGE=$(echo "${1}" | jq -r .image | envsubst) TMP_COMMAND=$(echo "${1}" | jq -r .command) echo "docker run ${DOCKERARGS} ${IMAGE} ${TMP_COMMAND}" From 96e699c4d76f8f701ad39235cb15e19021e2c8c1 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:17:17 -0700 Subject: [PATCH 06/83] fix: SC2162 shellcheck fixes. --- docker-entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 3d3eaba..08c14a3 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -184,7 +184,7 @@ function build_crontab() { rm -rf ${CRONTAB_FILE} ONSTART=() - while read i ; do + while read -r i ; do SCHEDULE=$(jq -r .["$i"].schedule "${CONFIG}" | sed 's/\*/\\*/g') if [ "${SCHEDULE}" == "null" ]; then @@ -228,7 +228,7 @@ EOF if [ "$(jq -r .["$i"].trigger "${CONFIG}")" != "null" ]; then - while read j ; do + while read -r j ; do if [ "$(jq .["$i"].trigger["$j"].command "${CONFIG}")" == "null" ]; then echo "Command Missing: $(jq -r .["$i"].trigger["$j"].command "${CONFIG}")" continue From dcd3e429f670698087916b8a06528e1e9445a8a0 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:17:58 -0700 Subject: [PATCH 07/83] fix: SC2166 shellcheck fixes. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index 08c14a3..14b3ef9 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -if [ -z "$DOCKER_HOST" -a "$DOCKER_PORT_2375_TCP" ]; then +if [ -z "$DOCKER_HOST" ] && [ -a "$DOCKER_PORT_2375_TCP" ]; then export DOCKER_HOST='tcp://docker:2375' fi From 6d92c31545fba5fcaf46c8197038be3cd197af9d Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:21:06 -0700 Subject: [PATCH 08/83] fix: SC2005 and SC2046 shellcheck fixes. --- docker-entrypoint | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 14b3ef9..cc7ec49 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -19,11 +19,11 @@ get_config() { if [ -f "${HOME_DIR}/config.json" ]; then jq 'map(.)' "${HOME_DIR}"/config.json > "${HOME_DIR}"/config.working.json elif [ -f "${HOME_DIR}/config.toml" ]; then - rq -t <<< $(cat "${HOME_DIR}"/config.toml) | jq 'map(.)' > "${HOME_DIR}"/config.json + rq -t <<< "$(cat "${HOME_DIR}"/config.toml)" | jq 'map(.)' > "${HOME_DIR}"/config.json elif [ -f "${HOME_DIR}/config.yml" ]; then - rq -y <<< $(cat "${HOME_DIR}"/config.yml) | jq 'map(.)' > "${HOME_DIR}"/config.json + rq -y <<< "$(cat "${HOME_DIR}"/config.yml)" | jq 'map(.)' > "${HOME_DIR}"/config.json elif [ -f "${HOME_DIR}/config.yaml" ]; then - rq -y <<< $(cat "${HOME_DIR}"/config.yaml) | jq 'map(.)' > "${HOME_DIR}"/config.json + rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)" | jq 'map(.)' > "${HOME_DIR}"/config.json fi } @@ -234,7 +234,7 @@ EOF continue fi #TRIGGER_COMMAND=$(make_cmd "$(jq -c .[$i].trigger[$j] ${CONFIG})") - echo "$(make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh + make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #COMMAND="${COMMAND} && ${TRIGGER_COMMAND}" done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi From 882fb137f29beac6a248efd2f8aa2a20c7f1f79b Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:25:09 -0700 Subject: [PATCH 09/83] fix: SC1001, SC2018, SC2019 shellcheck fixes. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index cc7ec49..8404445 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -55,7 +55,7 @@ ensure_docker_socket_accessible() { } slugify() { - echo "$@" | iconv -t ascii | sed -r s/[~\^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z + echo "$@" | iconv -t ascii | sed -r s/[~^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr '[:upper:]' '[:lower:]' } make_image_cmd() { From f18275eb0d3bc9b7d3a7a4a27d67842471d22afb Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:28:20 -0700 Subject: [PATCH 10/83] fix: simplifying logic. --- docker-entrypoint | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 8404445..1ddd1fd 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -16,15 +16,17 @@ if [ "${LOG_FILE}" == "" ]; then fi get_config() { + JSON_CONFIG={} if [ -f "${HOME_DIR}/config.json" ]; then - jq 'map(.)' "${HOME_DIR}"/config.json > "${HOME_DIR}"/config.working.json + JSON_CONFIG="$(cat "${HOME_DIR}"/config.json)" elif [ -f "${HOME_DIR}/config.toml" ]; then - rq -t <<< "$(cat "${HOME_DIR}"/config.toml)" | jq 'map(.)' > "${HOME_DIR}"/config.json + JSON_CONFIG="$(rq -t <<< "$(cat "${HOME_DIR}"/config.toml)")" elif [ -f "${HOME_DIR}/config.yml" ]; then - rq -y <<< "$(cat "${HOME_DIR}"/config.yml)" | jq 'map(.)' > "${HOME_DIR}"/config.json + JSON_CONFIG="$(rq -y <<< "$(cat "${HOME_DIR}"/config.yml)")" elif [ -f "${HOME_DIR}/config.yaml" ]; then - rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)" | jq 'map(.)' > "${HOME_DIR}"/config.json + JSON_CONFIG="$(rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)")" fi + jq 'map(.)' "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } DOCKER_SOCK=/var/run/docker.sock @@ -263,19 +265,13 @@ ensure_docker_socket_accessible start_app() { get_config - if [ -f "${HOME_DIR}/config.working.json" ]; then - export CONFIG=${HOME_DIR}/config.working.json - elif [ -f "${HOME_DIR}/config.json" ]; then - export CONFIG=${HOME_DIR}/config.json - else - echo "NO CONFIG FILE FOUND" + export CONFIG=${HOME_DIR}/config.working.json + if [ ! -f "${CONFIG}" ]; then + echo "Unable to find ${CONFIG}." + exit 1 fi - if [ "$1" = "crond" ]; then - if [ -f "${CONFIG}" ]; then - build_crontab - else - echo "Unable to find ${CONFIG}" - fi + if [ "$1" == "crond" ]; then + build_crontab fi echo "$@" exec "$@" From 99aa59ddb4153e192f0f904672f1cfab5d6f6a05 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:31:20 -0700 Subject: [PATCH 11/83] chore: whitespace and dead code cleanup. --- docker-entrypoint | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 1ddd1fd..9f406c6 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -5,7 +5,7 @@ if [ -z "$DOCKER_HOST" ] && [ -a "$DOCKER_PORT_2375_TCP" ]; then export DOCKER_HOST='tcp://docker:2375' fi -# for local testing only +# For local testing only. #HOME_DIR=. if [ "${LOG_FILE}" == "" ]; then @@ -32,25 +32,25 @@ get_config() { DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker -# Ensure dir exist - in case of volume mapping +# Ensure dir exist - in case of volume mapping. mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects ensure_docker_socket_accessible() { if ! grep -q "^docker:" /etc/group; then - # Ensure 'docker' user has permissions for docker socket (without changing permissions) + # Ensure 'docker' user has permissions for docker socket (without changing permissions). DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCK}) if [ "${DOCKER_GID}" != "0" ]; then if ! grep -qE "^[^:]+:[^:]+:${DOCKER_GID}:" /etc/group; then - # No group with such gid exists - create group docker + # No group with such gid exists - create group docker. addgroup -g "${DOCKER_GID}" docker adduser docker docker else - # Group with such gid exists - add user "docker" to this group + # Group with such gid exists - add user "docker" to this group. DOCKER_GROUP_NAME=$(getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}') adduser docker "$DOCKER_GROUP_NAME" fi else - # Docker socket belongs to "root" group - add user "docker" to this group + # Docker socket belongs to "root" group - add user "docker" to this group. adduser docker root fi fi @@ -65,12 +65,9 @@ make_image_cmd() { VOLUMES=$(echo "${1}" | jq -r '.volumes | map(" -v " + .) | join("")') PORTS=$(echo "${1}" | jq -r '.ports | map(" -p " + .) | join("")') EXPOSE=$(echo "${1}" | jq -r '.expose | map(" --expose " + .) | join("")') - # We'll add name in, if it exists NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network') ENVIRONMENT=$(echo "${1}" | jq -r '.environment | map(" -e " + .) | join("")') - # echo ${1} | jq -r '.environment | join("\n")' > ${PWD}/${NAME}.env - # ENVIRONMENT=" --env-file ${PWD}/${NAME}.env" if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi if [ -n "${NAME}" ]; then DOCKERARGS="${DOCKERARGS} --rm --name ${NAME} "; fi if [ -n "${NETWORK}" ]; then DOCKERARGS="${DOCKERARGS} --network ${NETWORK} "; fi @@ -93,8 +90,7 @@ make_container_cmd() { TMP_COMMAND=$(echo "${1}" | jq -r .command) if [ "${PROJECT}" != "null" ]; then - - # create bash script to detect all running containers + # Create bash script to detect all running containers. if [ "${SCRIPT_NAME}" == "null" ]; then SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) fi @@ -108,7 +104,6 @@ for CONTAINER_NAME in \$CONTAINERS; do done EOF echo "/bin/bash ${HOME_DIR}/projects/${SCRIPT_NAME}.sh" - # cat "/bin/bash ${HOME_DIR}/projects/${SCRIPT_NAME}.sh" else echo "docker exec ${DOCKERARGS} ${CONTAINER} ${TMP_COMMAND}" fi @@ -182,7 +177,6 @@ parse_schedule() { } function build_crontab() { - rm -rf ${CRONTAB_FILE} ONSTART=() @@ -216,28 +210,17 @@ cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #!/usr/bin/env bash set -e -# TODO find workaround -# [error] write /dev/stdout: broken pipe <- when using docker commands -#UUID=\$(cat /proc/sys/kernel/random/uuid) -#exec > >(read message; echo "\${UUID} \$(date -Iseconds) [info] \$message" | tee -a ${LOG_FILE} ) -#exec 2> >(read message; echo "\${UUID} \$(date -Iseconds) [error] \$message" | tee -a ${LOG_FILE} >&2) - echo "Start Cronjob **${SCRIPT_NAME}** ${COMMENT}" $(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF - - - if [ "$(jq -r .["$i"].trigger "${CONFIG}")" != "null" ]; then while read -r j ; do if [ "$(jq .["$i"].trigger["$j"].command "${CONFIG}")" == "null" ]; then echo "Command Missing: $(jq -r .["$i"].trigger["$j"].command "${CONFIG}")" continue fi - #TRIGGER_COMMAND=$(make_cmd "$(jq -c .[$i].trigger[$j] ${CONFIG})") make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh - #COMMAND="${COMMAND} && ${TRIGGER_COMMAND}" done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi @@ -260,7 +243,6 @@ EOF done } - ensure_docker_socket_accessible start_app() { From dc18a5199df675bdcf9ee4644250320dfcd8c6e8 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:33:41 -0700 Subject: [PATCH 12/83] chore: reorganization. --- docker-entrypoint | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 9f406c6..e9ece87 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -1,12 +1,19 @@ #!/usr/bin/env bash set -e +DOCKER_SOCK=/var/run/docker.sock +CRONTAB_FILE=/etc/crontabs/docker + +# For local testing only. +#HOME_DIR=. + +# Ensure dir exist - in case of volume mapping. +mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects + if [ -z "$DOCKER_HOST" ] && [ -a "$DOCKER_PORT_2375_TCP" ]; then export DOCKER_HOST='tcp://docker:2375' fi -# For local testing only. -#HOME_DIR=. if [ "${LOG_FILE}" == "" ]; then LOG_DIR=/var/log/crontab @@ -29,12 +36,6 @@ get_config() { jq 'map(.)' "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } -DOCKER_SOCK=/var/run/docker.sock -CRONTAB_FILE=/etc/crontabs/docker - -# Ensure dir exist - in case of volume mapping. -mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects - ensure_docker_socket_accessible() { if ! grep -q "^docker:" /etc/group; then # Ensure 'docker' user has permissions for docker socket (without changing permissions). @@ -243,8 +244,6 @@ EOF done } -ensure_docker_socket_accessible - start_app() { get_config export CONFIG=${HOME_DIR}/config.working.json @@ -259,4 +258,5 @@ start_app() { exec "$@" } +ensure_docker_socket_accessible start_app "$@" From c685687367918b3b0b62e6775776ded1994a7756 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:34:43 -0700 Subject: [PATCH 13/83] chore: rename function to better describe what it does. --- docker-entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index e9ece87..6dac6af 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -22,7 +22,7 @@ if [ "${LOG_FILE}" == "" ]; then touch ${LOG_FILE} fi -get_config() { +normalize_config() { JSON_CONFIG={} if [ -f "${HOME_DIR}/config.json" ]; then JSON_CONFIG="$(cat "${HOME_DIR}"/config.json)" @@ -245,7 +245,7 @@ EOF } start_app() { - get_config + normalize_config export CONFIG=${HOME_DIR}/config.working.json if [ ! -f "${CONFIG}" ]; then echo "Unable to find ${CONFIG}." From 84265413ee79bb51c5e81a2d1555c2d567cfafc4 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:38:01 -0700 Subject: [PATCH 14/83] chore: adding error handling. --- docker-entrypoint | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-entrypoint b/docker-entrypoint index 6dac6af..b5a7fd7 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -7,6 +7,11 @@ CRONTAB_FILE=/etc/crontabs/docker # For local testing only. #HOME_DIR=. +if [ -z "${HOME_DIR}" ]; then + echo "HOME_DIR not set." + exit 1 +fi + # Ensure dir exist - in case of volume mapping. mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects From f0c30632bd919fa159db8095cbca579f20446164 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:43:14 -0700 Subject: [PATCH 15/83] chore: standardize on {}. --- docker-entrypoint | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index b5a7fd7..3987ea5 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -15,7 +15,7 @@ fi # Ensure dir exist - in case of volume mapping. mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects -if [ -z "$DOCKER_HOST" ] && [ -a "$DOCKER_PORT_2375_TCP" ]; then +if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST='tcp://docker:2375' fi @@ -53,7 +53,7 @@ ensure_docker_socket_accessible() { else # Group with such gid exists - add user "docker" to this group. DOCKER_GROUP_NAME=$(getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}') - adduser docker "$DOCKER_GROUP_NAME" + adduser docker "${DOCKER_GROUP_NAME}" fi else # Docker socket belongs to "root" group - add user "docker" to this group. @@ -63,7 +63,7 @@ ensure_docker_socket_accessible() { } slugify() { - echo "$@" | iconv -t ascii | sed -r s/[~^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr '[:upper:]' '[:lower:]' + echo "${@}" | iconv -t ascii | sed -r s/[~^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr '[:upper:]' '[:lower:]' } make_image_cmd() { @@ -90,7 +90,7 @@ make_container_cmd() { DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi SCRIPT_NAME=$(echo "${1}" | jq -r .name) - SCRIPT_NAME=$(slugify "$SCRIPT_NAME") + SCRIPT_NAME=$(slugify "${SCRIPT_NAME}") PROJECT=$(echo "${1}" | jq -r .project) CONTAINER=$(echo "${1}" | jq -r .container | envsubst) TMP_COMMAND=$(echo "${1}" | jq -r .command) @@ -123,11 +123,11 @@ EOF make_cmd() { if [ "$(echo "${1}" | jq -r .image)" != "null" ]; then - make_image_cmd "$1" + make_image_cmd "${1}" elif [ "$(echo "${1}" | jq -r .container)" != "null" ]; then - make_container_cmd "$1" + make_container_cmd "${1}" #elif [ "$(echo ${1} | jq -r .host)" != "null" ]; then - # make_host_cmd "$1" + # make_host_cmd "${1}" else echo "${1}" | jq -r .command fi @@ -160,9 +160,9 @@ parse_schedule() { TIME=$2 TOTAL=0 - M=$(echo "$TIME" | grep -o '[0-9]\+m') - H=$(echo "$TIME" | grep -o '[0-9]\+h') - D=$(echo "$TIME" | grep -o '[0-9]\+d') + M=$(echo "${TIME}" | grep -o '[0-9]\+m') + H=$(echo "${TIME}" | grep -o '[0-9]\+h') + D=$(echo "${TIME}" | grep -o '[0-9]\+d') if [ -n "${M}" ]; then TOTAL=$((TOTAL + ${M::-1})) @@ -206,7 +206,7 @@ function build_crontab() { fi SCRIPT_NAME=$(jq -r .["$i"].name "${CONFIG}") - SCRIPT_NAME=$(slugify "$SCRIPT_NAME") + SCRIPT_NAME=$(slugify "${SCRIPT_NAME}") if [ "${SCRIPT_NAME}" == "null" ]; then SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) fi @@ -256,12 +256,12 @@ start_app() { echo "Unable to find ${CONFIG}." exit 1 fi - if [ "$1" == "crond" ]; then + if [ "${1}" == "crond" ]; then build_crontab fi - echo "$@" - exec "$@" + echo "${@}" + exec "${@}" } ensure_docker_socket_accessible -start_app "$@" +start_app "${@}" From 7f29f0621e1410580f9e75d3de6d63cbd457d9a7 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:51:38 -0700 Subject: [PATCH 16/83] chore: standardizing on env vars. --- docker-entrypoint | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 3987ea5..e44ded5 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -195,8 +195,9 @@ function build_crontab() { fi SCHEDULE=$(parse_schedule "${SCHEDULE}" | sed 's/\\//g') - if [ "$(jq -r .["$i"].command "${CONFIG}")" == "null" ]; then - echo "Command Missing: $(jq -r .["$i"].command "${CONFIG}")" + COMMAND=$(jq -r .["$i"].command "${CONFIG}") + if [ "${COMMAND}" == "null" ]; then + echo "Command Missing: '${COMMAND}'" continue fi @@ -220,10 +221,12 @@ echo "Start Cronjob **${SCRIPT_NAME}** ${COMMENT}" $(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF - if [ "$(jq -r .["$i"].trigger "${CONFIG}")" != "null" ]; then + TRIGGER=$(jq -r .["$i"].trigger "${CONFIG}") + if [ "${TRIGGER}" != "null" ]; then while read -r j ; do - if [ "$(jq .["$i"].trigger["$j"].command "${CONFIG}")" == "null" ]; then - echo "Command Missing: $(jq -r .["$i"].trigger["$j"].command "${CONFIG}")" + TRIGGER_COMMAND=$(jq .["$i"].trigger["$j"].command "${CONFIG}") + if [ "${TRIGGER_COMMAND}" == "null" ]; then + echo "Command Missing: '${TRIGGER_COMMAND}'" continue fi make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh From 1d844f8318e524e23b22462e5845a38112066524 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 08:58:33 -0700 Subject: [PATCH 17/83] chore: missed one spot using previously set env var. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index e44ded5..1c7a91d 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -229,7 +229,7 @@ EOF echo "Command Missing: '${TRIGGER_COMMAND}'" continue fi - make_cmd "$(jq -c .["$i"].trigger["$j"] "${CONFIG}")" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh + make_cmd "${TRIGGER_COMMAND}" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi From 8f1d9200e070ab517b5f78f42390347e68dd5d8e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 09:21:04 -0700 Subject: [PATCH 18/83] chore: tabs to spaces. --- docker-entrypoint | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 1c7a91d..6a28e96 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -16,14 +16,14 @@ fi mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then - export DOCKER_HOST='tcp://docker:2375' + export DOCKER_HOST='tcp://docker:2375' fi if [ "${LOG_FILE}" == "" ]; then LOG_DIR=/var/log/crontab - LOG_FILE=${LOG_DIR}/jobs.log - mkdir -p ${LOG_DIR} + LOG_FILE=${LOG_DIR}/jobs.log + mkdir -p ${LOG_DIR} touch ${LOG_FILE} fi @@ -134,7 +134,7 @@ make_cmd() { } parse_schedule() { - case $1 in + case $1 in "@yearly") echo "0 0 1 1 *" ;; From c4da20ce834729d9ada2f0616baed5a02a3b33bd Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 09:29:04 -0700 Subject: [PATCH 19/83] fix: addressing issue with parsing json. --- docker-entrypoint | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 6a28e96..272f182 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -19,7 +19,6 @@ if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST='tcp://docker:2375' fi - if [ "${LOG_FILE}" == "" ]; then LOG_DIR=/var/log/crontab LOG_FILE=${LOG_DIR}/jobs.log @@ -38,7 +37,7 @@ normalize_config() { elif [ -f "${HOME_DIR}/config.yaml" ]; then JSON_CONFIG="$(rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)")" fi - jq 'map(.)' "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json + jq -r 'map(.)' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } ensure_docker_socket_accessible() { From 36ef24f15ff5ee5d283e2297721fd7208d65ed31 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 09:48:40 -0700 Subject: [PATCH 20/83] chore: omit additional items. --- .gitignore | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f8a3aa0..f6473b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ .idea *.iml - -config.json .vscode .DS_Store + +config.json +config.working.json + +jobs/ +projects/ From a523ad7d747d942383c3c644fbcf8a8abc4d82a7 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 09:49:17 -0700 Subject: [PATCH 21/83] fix: reset COMMENT. --- docker-entrypoint | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 272f182..41af427 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -186,7 +186,6 @@ function build_crontab() { ONSTART=() while read -r i ; do - SCHEDULE=$(jq -r .["$i"].schedule "${CONFIG}" | sed 's/\*/\\*/g') if [ "${SCHEDULE}" == "null" ]; then echo "Schedule Missing: $(jq -r .["$i"].schedule "${CONFIG}")" @@ -203,6 +202,9 @@ function build_crontab() { COMMENT=$(jq -r .["$i"].comment "${CONFIG}") if [ "${COMMENT}" != "null" ]; then echo "# ${COMMENT}" >> ${CRONTAB_FILE} + else + # Reset COMMENT to empty rather than keep the 'null' value. + COMMENT= fi SCRIPT_NAME=$(jq -r .["$i"].name "${CONFIG}") @@ -217,7 +219,6 @@ cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh set -e echo "Start Cronjob **${SCRIPT_NAME}** ${COMMENT}" - $(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF TRIGGER=$(jq -r .["$i"].trigger "${CONFIG}") From 9316c9f5c2dd05c329897ae28b3aa2eb57f8a047 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 10:22:43 -0700 Subject: [PATCH 22/83] fix: address jq parsing errors when some properties are not set. --- docker-entrypoint | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 41af427..1d51a53 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -67,19 +67,19 @@ slugify() { make_image_cmd() { DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) - VOLUMES=$(echo "${1}" | jq -r '.volumes | map(" -v " + .) | join("")') - PORTS=$(echo "${1}" | jq -r '.ports | map(" -p " + .) | join("")') - EXPOSE=$(echo "${1}" | jq -r '.expose | map(" --expose " + .) | join("")') + VOLUMES=$(echo "${1}" | jq -r 'select(.volumes != null) | .volumes | map(" -v " + .) | join("")') + PORTS=$(echo "${1}" | jq -r 'select(.ports != null) | .ports | map(" -p " + .) | join("")') + EXPOSE=$(echo "${1}" | jq -r 'select(.expose != null) | .expose | map(" --expose " + .) | join("")') NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network') - ENVIRONMENT=$(echo "${1}" | jq -r '.environment | map(" -e " + .) | join("")') + ENVIRONMENT=$(echo "${1}" | jq -r 'select(.environment != null) | .environment | map(" -e " + .) | join("")') if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi - if [ -n "${NAME}" ]; then DOCKERARGS="${DOCKERARGS} --rm --name ${NAME} "; fi - if [ -n "${NETWORK}" ]; then DOCKERARGS="${DOCKERARGS} --network ${NETWORK} "; fi - if [ -n "${VOLUMES}" ]; then DOCKERARGS="${DOCKERARGS}${VOLUMES}"; fi - if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS="${DOCKERARGS}${ENVIRONMENT}"; fi - if [ -n "${PORTS}" ]; then DOCKERARGS="${DOCKERARGS}${PORTS}"; fi - if [ -n "${EXPOSE}" ]; then DOCKERARGS="${DOCKERARGS}${EXPOSE}"; fi + if [ -n "${NAME}" ]; then DOCKERARGS+=" --rm --name ${NAME} "; fi + if [ -n "${NETWORK}" ]; then DOCKERARGS+=" --network ${NETWORK} "; fi + if [ -n "${VOLUMES}" ]; then DOCKERARGS+="${VOLUMES}"; fi + if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS+="${ENVIRONMENT}"; fi + if [ -n "${PORTS}" ]; then DOCKERARGS+="${PORTS}"; fi + if [ -n "${EXPOSE}" ]; then DOCKERARGS+="${EXPOSE}"; fi IMAGE=$(echo "${1}" | jq -r .image | envsubst) TMP_COMMAND=$(echo "${1}" | jq -r .command) echo "docker run ${DOCKERARGS} ${IMAGE} ${TMP_COMMAND}" From 0643e9ceaeee2c8df64207dad50887c11d558081 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 10:40:02 -0700 Subject: [PATCH 23/83] chore: moving line to be consistent. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index 1d51a53..09ef079 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -67,13 +67,13 @@ slugify() { make_image_cmd() { DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) + if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi VOLUMES=$(echo "${1}" | jq -r 'select(.volumes != null) | .volumes | map(" -v " + .) | join("")') PORTS=$(echo "${1}" | jq -r 'select(.ports != null) | .ports | map(" -p " + .) | join("")') EXPOSE=$(echo "${1}" | jq -r 'select(.expose != null) | .expose | map(" --expose " + .) | join("")') NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network') ENVIRONMENT=$(echo "${1}" | jq -r 'select(.environment != null) | .environment | map(" -e " + .) | join("")') - if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi if [ -n "${NAME}" ]; then DOCKERARGS+=" --rm --name ${NAME} "; fi if [ -n "${NETWORK}" ]; then DOCKERARGS+=" --network ${NETWORK} "; fi if [ -n "${VOLUMES}" ]; then DOCKERARGS+="${VOLUMES}"; fi From 08bbceff959460d63b77fcc0746dd14b5005afce Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 10:43:25 -0700 Subject: [PATCH 24/83] fix: cleaning script creation. --- docker-entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 09ef079..fc3deca 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -104,8 +104,8 @@ cat << EOF > "${HOME_DIR}"/projects/"${SCRIPT_NAME}".sh set -e CONTAINERS=\$(docker ps --format '{{.Names}}' | grep -E "^${PROJECT}_${CONTAINER}.[0-9]+") -for CONTAINER_NAME in \$CONTAINERS; do - docker exec ${DOCKERARGS} \${CONTAINER_NAME} ${TMP_COMMAND} +for CONTAINER_NAME in \${CONTAINERS}; do + docker exec "${DOCKERARGS} \${CONTAINER_NAME} ${TMP_COMMAND}" done EOF echo "/bin/bash ${HOME_DIR}/projects/${SCRIPT_NAME}.sh" From e69b2767074136107d45a6f4fadfd3a1cf26fe16 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 10:45:41 -0700 Subject: [PATCH 25/83] chore: nicer formatting. --- docker-entrypoint | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index fc3deca..4509cb3 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -201,10 +201,11 @@ function build_crontab() { COMMENT=$(jq -r .["$i"].comment "${CONFIG}") if [ "${COMMENT}" != "null" ]; then - echo "# ${COMMENT}" >> ${CRONTAB_FILE} + COMMENT=" ${COMMENT}" + echo "#${COMMENT}" >> ${CRONTAB_FILE} else # Reset COMMENT to empty rather than keep the 'null' value. - COMMENT= + COMMENT=" " fi SCRIPT_NAME=$(jq -r .["$i"].name "${CONFIG}") @@ -218,7 +219,7 @@ cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #!/usr/bin/env bash set -e -echo "Start Cronjob **${SCRIPT_NAME}** ${COMMENT}" +echo "Start Cronjob **${SCRIPT_NAME}**${COMMENT}" $(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF TRIGGER=$(jq -r .["$i"].trigger "${CONFIG}") @@ -233,7 +234,7 @@ EOF done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi - echo "echo \"End Cronjob **${SCRIPT_NAME}** ${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh + echo "echo \"End Cronjob **${SCRIPT_NAME}**${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh echo "${SCHEDULE} ${COMMAND}" >> ${CRONTAB_FILE} From 31d0b732b2ae29069edb6a93af60b9fb47c68603 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 10:53:34 -0700 Subject: [PATCH 26/83] chore: cleaning up echos. --- docker-entrypoint | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-entrypoint b/docker-entrypoint index 4509cb3..de76ba9 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -16,7 +16,7 @@ fi mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then - export DOCKER_HOST='tcp://docker:2375' + export DOCKER_HOST="tcp://docker:2375" fi if [ "${LOG_FILE}" == "" ]; then @@ -188,14 +188,14 @@ function build_crontab() { while read -r i ; do SCHEDULE=$(jq -r .["$i"].schedule "${CONFIG}" | sed 's/\*/\\*/g') if [ "${SCHEDULE}" == "null" ]; then - echo "Schedule Missing: $(jq -r .["$i"].schedule "${CONFIG}")" + echo "'schedule' missing: $(jq -r .["$i"].schedule "${CONFIG}")" continue fi SCHEDULE=$(parse_schedule "${SCHEDULE}" | sed 's/\\//g') COMMAND=$(jq -r .["$i"].command "${CONFIG}") if [ "${COMMAND}" == "null" ]; then - echo "Command Missing: '${COMMAND}'" + echo "'command' missing: '${COMMAND}'" continue fi @@ -219,7 +219,7 @@ cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh #!/usr/bin/env bash set -e -echo "Start Cronjob **${SCRIPT_NAME}**${COMMENT}" +echo "start cron job **${SCRIPT_NAME}**${COMMENT}" $(make_cmd "$(jq -c .["$i"] "${CONFIG}")") EOF TRIGGER=$(jq -r .["$i"].trigger "${CONFIG}") @@ -227,14 +227,14 @@ EOF while read -r j ; do TRIGGER_COMMAND=$(jq .["$i"].trigger["$j"].command "${CONFIG}") if [ "${TRIGGER_COMMAND}" == "null" ]; then - echo "Command Missing: '${TRIGGER_COMMAND}'" + echo "'command' missing: '${TRIGGER_COMMAND}'" continue fi make_cmd "${TRIGGER_COMMAND}" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") fi - echo "echo \"End Cronjob **${SCRIPT_NAME}**${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh + echo "echo \"end cron job **${SCRIPT_NAME}**${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh echo "${SCHEDULE} ${COMMAND}" >> ${CRONTAB_FILE} @@ -257,7 +257,7 @@ start_app() { normalize_config export CONFIG=${HOME_DIR}/config.working.json if [ ! -f "${CONFIG}" ]; then - echo "Unable to find ${CONFIG}." + echo "generated ${CONFIG} missing." exit 1 fi if [ "${1}" == "crond" ]; then From 9da424985d018ead48b360539cc1556137204003 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 16 Jul 2022 11:24:32 -0700 Subject: [PATCH 27/83] fix: moving the top level key name to the "name" property. --- docker-entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint b/docker-entrypoint index de76ba9..dd8ada5 100755 --- a/docker-entrypoint +++ b/docker-entrypoint @@ -37,7 +37,7 @@ normalize_config() { elif [ -f "${HOME_DIR}/config.yaml" ]; then JSON_CONFIG="$(rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)")" fi - jq -r 'map(.)' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json + jq -r 'to_entries | map_values(.value + { name: .key })' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } ensure_docker_socket_accessible() { From 76b524fd170894e1716232ff0f5224e2466293f8 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 7 Aug 2022 08:14:21 -0700 Subject: [PATCH 28/83] chore: merge from private git repo. --- .github/workflows/build.yml | 51 ++++++++++++++---------------- Dockerfile | 40 +++++++++++++++-------- README.md | 48 +--------------------------- docker-entrypoint => entrypoint.sh | 17 +++------- test_logging | 20 ------------ 5 files changed, 54 insertions(+), 122 deletions(-) rename docker-entrypoint => entrypoint.sh (96%) delete mode 100755 test_logging diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b80b36c..7d524eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,37 +13,32 @@ jobs: multi: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - if: github.ref == 'refs/heads/main' - name: Conditional(Set tag as `latest`) - run: echo "tag=willfarrell/crontab:latest" >> $GITHUB_ENV +# - name: Login to DockerHub +# uses: docker/login-action@v1 +# with: +# username: ${{ secrets.DOCKER_USERNAME }} +# password: ${{ secrets.DOCKER_PASSWORD }} + +# - if: github.ref == 'refs/heads/main' +# name: Conditional(Set tag as `latest`) +# run: echo "tag=willfarrell/crontab:latest" >> $GITHUB_ENV - - if: startsWith(github.ref, 'refs/tags/') - name: Conditional(Set tag as `{version}`) - run: echo "tag=willfarrell/crontab:${GITHUB_REF#refs/*/}" >> $GITHUB_ENV +# - if: startsWith(github.ref, 'refs/tags/') +# name: Conditional(Set tag as `{version}`) +# run: echo "tag=willfarrell/crontab:${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - tags: | - ${{ env.tag }} +# - name: Build and push +# uses: docker/build-push-action@v2 +# with: +# context: . +# file: ./Dockerfile +# push: true +# tags: | +# ${{ env.tag }} diff --git a/Dockerfile b/Dockerfile index a5dfb47..d30258e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,36 @@ -FROM alpine:3.16 as rq-build +FROM alpine:latest as rq-build ENV RQ_VERSION=1.0.2 -WORKDIR /root/ +WORKDIR /usr/bin/rq/ -RUN apk --update add upx \ - && wget https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz \ - && tar -xvf rq-v1.0.2-x86_64-unknown-linux-musl.tar.gz \ - && upx --brute rq +RUN apk update && \ + apk upgrade && \ + apk add --no-cache \ + upx && \ + wget https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ + tar -xvf rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ + upx --brute rq -FROM library/docker:stable - -COPY --from=rq-build /root/rq /usr/local/bin +FROM docker:latest as release ENV HOME_DIR=/opt/crontab -RUN apk add --no-cache --virtual .run-deps gettext jq bash tini \ - && mkdir -p ${HOME_DIR}/jobs ${HOME_DIR}/projects \ - && adduser -S docker -D -COPY docker-entrypoint / -ENTRYPOINT ["/sbin/tini", "--", "/docker-entrypoint"] +RUN apk update && \ + apk upgrade && \ + apk add --no-cache \ + bash \ + curl \ + gettext \ + jq \ + tini \ + wget && \ + mkdir -p ${HOME_DIR}/jobs ${HOME_DIR}/projects && \ + adduser -S docker -D + +COPY --from=rq-build /usr/bin/rq/rq /usr/local/bin +COPY entrypoint.sh / + +ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 diff --git a/README.md b/README.md index 8fc98e0..0caf294 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,6 @@ # docker-crontab -A simple wrapper over `docker` to all complex cron job to be run in other containers. - -## Supported tags and Dockerfile links - -- [`latest` (*Dockerfile*)](https://github.com/willfarrell/docker-crontab/blob/master/Dockerfile) -- [`1.0.0` (*Dockerfile*)](https://github.com/willfarrell/docker-crontab/blob/1.0.0/Dockerfile) -- [`0.6.0` (*Dockerfile*)](https://github.com/willfarrell/docker-crontab/blob/0.6.0/Dockerfile) - -![](https://img.shields.io/docker/pulls/willfarrell/crontab "Total docker pulls") [![](https://images.microbadger.com/badges/image/willfarrell/crontab.svg)](http://microbadger.com/images/willfarrell/crontab "Get your own image badge on microbadger.com") +A simple wrapper over `docker` to all complex cron job to be run in other containers. Note, this is a maintained fork of [willfarrell/docker-crontab](https://github.com/willfarrell/docker-crontab). ## Why? Yes, I'm aware of [mcuadros/ofelia](https://github.com/mcuadros/ofelia) (>250MB when this was created), it was the main inspiration for this project. @@ -23,7 +15,6 @@ A great project, don't get me wrong. It was just missing certain key enterprise - Ability to trigger scripts in other containers on completion cron job using `trigger`. ## Config file - The config file can be specifed in any of `json`, `toml`, or `yaml`, and can be defined as either an array or mapping (top-level keys will be ignored; can be useful for organizing commands) - `name`: Human readable name that will be used as the job filename. Will be converted into a slug. Optional. @@ -61,7 +52,6 @@ See [`config-samples`](config-samples) for examples. ## How to use ### Command Line - ```bash docker build -t crontab . docker run -d \ @@ -73,7 +63,6 @@ docker run -d \ ``` ### Use with docker-compose - 1. Figure out which network name used for your docker-compose containers * use `docker network ls` to see existing networks * if your `docker-compose.yml` is in `my_dir` directory, you probably has network `my_dir_default` @@ -82,38 +71,3 @@ docker run -d \ * use `--network NETWORK_NAME` to connect new container into docker-compose network * use `--rm --name NAME` to use named container * e.g. `"dockerargs": "--network my_dir_default --rm --name my-best-cron-job"` - -### Dockerfile - -```Dockerfile -FROM willfarrell/crontab - -COPY config.json ${HOME_DIR}/ - -``` - -### Logrotate Dockerfile - -```Dockerfile -FROM willfarrell/crontab - -RUN apk add --no-cache logrotate -RUN echo "*/5 * * * * /usr/sbin/logrotate /etc/logrotate.conf" >> /etc/crontabs/logrotate -COPY logrotate.conf /etc/logrotate.conf - -CMD ["crond", "-f"] -``` - -### Logging - In Dev - -All `stdout` is captured, formatted, and saved to `/var/log/crontab/jobs.log`. Set `LOG_FILE` to `/dev/null` to disable logging. - -example: `e6ced859-1563-493b-b1b1-5a190b29e938 2017-06-18T01:27:10+0000 [info] Start Cronjob **map-a-vol** map a volume` - -grok: `CRONTABLOG %{DATA:request_id} %{TIMESTAMP_ISO8601:timestamp} \[%{LOGLEVEL:severity}\] %{GREEDYDATA:message}` - -## TODO -- [ ] Have ability to auto regenerate crontab on file change (signal HUP?) -- [ ] Run commands on host machine (w/ --privileged?) -- [ ] Write tests -- [ ] Setup TravisCI diff --git a/docker-entrypoint b/entrypoint.sh similarity index 96% rename from docker-entrypoint rename to entrypoint.sh index dd8ada5..c3e0aac 100755 --- a/docker-entrypoint +++ b/entrypoint.sh @@ -1,12 +1,10 @@ -#!/usr/bin/env bash +#!/bin/bash + set -e DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker -# For local testing only. -#HOME_DIR=. - if [ -z "${HOME_DIR}" ]; then echo "HOME_DIR not set." exit 1 @@ -114,19 +112,11 @@ EOF fi } -#make_host_cmd() { -# HOST_BINARY=$(echo ${1} | jq -r .host) -# TMP_COMMAND=$(echo ${1} | jq -r .command) -# echo "${HOST_BINARY} ${TMP_COMMAND}" -#} - make_cmd() { if [ "$(echo "${1}" | jq -r .image)" != "null" ]; then make_image_cmd "${1}" elif [ "$(echo "${1}" | jq -r .container)" != "null" ]; then make_container_cmd "${1}" - #elif [ "$(echo ${1} | jq -r .host)" != "null" ]; then - # make_host_cmd "${1}" else echo "${1}" | jq -r .command fi @@ -257,7 +247,7 @@ start_app() { normalize_config export CONFIG=${HOME_DIR}/config.working.json if [ ! -f "${CONFIG}" ]; then - echo "generated ${CONFIG} missing." + echo "generated ${CONFIG} missing. exiting." exit 1 fi if [ "${1}" == "crond" ]; then @@ -268,4 +258,5 @@ start_app() { } ensure_docker_socket_accessible +printf "โœจ starting crontab container โœจ\n" start_app "${@}" diff --git a/test_logging b/test_logging deleted file mode 100755 index 97d5645..0000000 --- a/test_logging +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This file is for testing the logging of docker output #8 - -LOG_FILE=./jobs.log -touch ${LOG_FILE} -UUID="xxxxxxxxxxxxxxxxx" - -exec > >(read message; echo "${UUID} $(date) [info] $message" | tee -a ${LOG_FILE} ) -exec 2> >(read message; echo "${UUID} $(date) [error] $message" | tee -a ${LOG_FILE} >&2) - -echo "Start" - -docker run alpine sh -c 'while :; do echo "ping"; sleep 1; done' -# [error] write /dev/stdout: broken pipe -# --log-driver syslog <- errors -# --log-driver none <- errors - -echo "End" From 035cd4a9061239e8aa2286da8f1db33bd3e6c212 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 18 Feb 2023 18:42:20 -0800 Subject: [PATCH 29/83] feat: Major Updates - removed support for `projects` as the feature was very incomplete and it served little purpose - added support for common settings between jobs using `~~shared-settings` as a key in the config - cleaned up some items that have long bugged me - better reuse of code - better variable naming - improved flow and readability - formatting to the logs --- Dockerfile | 3 +- README.md | 74 +++++++++++++++--------- entrypoint.sh | 155 +++++++++++++++++++++++++------------------------- 3 files changed, 127 insertions(+), 105 deletions(-) diff --git a/Dockerfile b/Dockerfile index d30258e..4b84154 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,8 @@ RUN apk update && \ jq \ tini \ wget && \ - mkdir -p ${HOME_DIR}/jobs ${HOME_DIR}/projects && \ + mkdir -p ${HOME_DIR}/jobs && \ + rm -rf /etc/periodic /etc/crontabs/root && \ adduser -S docker -D COPY --from=rq-build /usr/bin/rq/rq /usr/local/bin diff --git a/README.md b/README.md index 0caf294..5a625ca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# docker-crontab +# crontab -A simple wrapper over `docker` to all complex cron job to be run in other containers. Note, this is a maintained fork of [willfarrell/docker-crontab](https://github.com/willfarrell/docker-crontab). +A simple wrapper over `docker` to all complex cron job to be run in other containers. ## Why? Yes, I'm aware of [mcuadros/ofelia](https://github.com/mcuadros/ofelia) (>250MB when this was created), it was the main inspiration for this project. @@ -11,42 +11,44 @@ A great project, don't get me wrong. It was just missing certain key enterprise - Allows for comments, cause we all need friendly reminders of what `update_script.sh` actually does. - Start an image using `image`. - Run command in a container using `container`. -- Run command on a instances of a scaled container using `project`. - Ability to trigger scripts in other containers on completion cron job using `trigger`. +- Ability to share settings between cron jobs using `~~shared-settings` as a key. ## Config file -The config file can be specifed in any of `json`, `toml`, or `yaml`, and can be defined as either an array or mapping (top-level keys will be ignored; can be useful for organizing commands) +The config file can be specified in any of `json`, `toml`, or `yaml`, and can be defined as either an array or mapping (top-level keys will be ignored; can be useful for organizing commands) - `name`: Human readable name that will be used as the job filename. Will be converted into a slug. Optional. - `comment`: Comments to be included with crontab entry. Optional. -- `schedule`: Crontab schedule syntax as described in https://en.wikipedia.org/wiki/Cron. Ex `@hourly`, `@every 1h30m`, `* * * * *`. Required. +- `schedule`: Crontab schedule syntax as described in https://en.wikipedia.org/wiki/Cron. Examples: `@hourly`, `@every 1h30m`, `* * * * *`. Required. - `command`: Command to be run on in crontab container or docker container/image. Required. - `image`: Docker images name (ex `library/alpine:3.5`). Optional. -- `project`: Docker Compose/Swarm project name. Optional, only applies when `contain` is included. -- `container`: Full container name or container alias if `project` is set. Ignored if `image` is included. Optional. +- `container`: Full container name. Ignored if `image` is included. Optional. - `dockerargs`: Command line docker `run`/`exec` arguments for full control. Defaults to ` `. -- `trigger`: Array of docker-crontab subset objects. Subset includes: `image`,`project`,`container`,`command`,`dockerargs` -- `onstart`: Run the command on `crontab` container start, set to `true`. Optional, defaults to falsey. +- `trigger`: Array of docker-crontab subset objects. Sub-set includes: `image`, `container`, `command`, `dockerargs` +- `onstart`: Run the command on `crontab` container start, set to `true`. Optional, defaults to false. See [`config-samples`](config-samples) for examples. ```json -[{ - "schedule":"@every 5m", - "command":"/usr/sbin/logrotate /etc/logrotate.conf" - },{ - "comment":"Regenerate Certificate then reload nginx", - "schedule":"43 6,18 * * *", - "command":"sh -c 'dehydrated --cron --out /etc/ssl --domain ${LE_DOMAIN} --challenge dns-01 --hook dehydrated-dns'", - "dockerargs":"--env-file /opt/crontab/env/letsencrypt.env -v webapp_nginx_tls_cert:/etc/ssl -v webapp_nginx_acme_challenge:/var/www/.well-known/acme-challenge", - "image":"willfarrell/letsencrypt", - "trigger":[{ - "command":"sh -c '/etc/scripts/make_hpkp ${NGINX_DOMAIN} && /usr/sbin/nginx -t && /usr/sbin/nginx -s reload'", - "project":"conduit", - "container":"nginx" - }], - "onstart":true - }] +{ + "logrotate": { + "schedule":"@every 5m", + "command":"/usr/sbin/logrotate /etc/logrotate.conf" + }, + "cert-regen": { + "comment":"Regenerate Certificate then reload nginx", + "schedule":"43 6,18 * * *", + "command":"sh -c 'dehydrated --cron --out /etc/ssl --domain ${LE_DOMAIN} --challenge dns-01 --hook dehydrated-dns'", + "dockerargs":"--it --env-file /opt/crontab/env/letsencrypt.env", + "volumes":["webapp_nginx_tls_cert:/etc/ssl", "webapp_nginx_acme_challenge:/var/www/.well-known/acme-challenge"], + "image":"willfarrell/letsencrypt", + "trigger":[{ + "command":"sh -c '/etc/scripts/make_hpkp ${NGINX_DOMAIN} && /usr/sbin/nginx -t && /usr/sbin/nginx -s reload'", + "container":"nginx" + }], + "onstart":true + } +} ``` ## How to use @@ -69,5 +71,23 @@ docker run -d \ * otherwise [read the docker-compose docs](https://docs.docker.com/compose/networking/) 2. Add `dockerargs` to your docker-crontab `config.json` * use `--network NETWORK_NAME` to connect new container into docker-compose network - * use `--rm --name NAME` to use named container - * e.g. `"dockerargs": "--network my_dir_default --rm --name my-best-cron-job"` + * use `--name NAME` to use named container + * e.g. `"dockerargs": "--it"` + +### Dockerfile +```Dockerfile +FROM registry.gitlab.com/simplicityguy/docker/crontab + +COPY config.json ${HOME_DIR}/ +``` + +### Logrotate Dockerfile +```Dockerfile +FROM registry.gitlab.com/simplicityguy/docker/crontab + +RUN apk add --no-cache logrotate +RUN echo "*/5 * * * * /usr/sbin/logrotate /etc/logrotate.conf" >> /etc/crontabs/logrotate +COPY logrotate.conf /etc/logrotate.conf + +CMD ["crond", "-f"] +``` diff --git a/entrypoint.sh b/entrypoint.sh index c3e0aac..37e8b87 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,7 +11,7 @@ if [ -z "${HOME_DIR}" ]; then fi # Ensure dir exist - in case of volume mapping. -mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/projects +mkdir -p "${HOME_DIR}"/jobs if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST="tcp://docker:2375" @@ -35,7 +35,8 @@ normalize_config() { elif [ -f "${HOME_DIR}/config.yaml" ]; then JSON_CONFIG="$(rq -y <<< "$(cat "${HOME_DIR}"/config.yaml)")" fi - jq -r 'to_entries | map_values(.value + { name: .key })' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json + + jq -S -r '."~~shared-settings" as $shared | del(."~~shared-settings") | to_entries | map_values(.value + { name: .key } + $shared)' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } ensure_docker_socket_accessible() { @@ -44,16 +45,16 @@ ensure_docker_socket_accessible() { DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCK}) if [ "${DOCKER_GID}" != "0" ]; then if ! grep -qE "^[^:]+:[^:]+:${DOCKER_GID}:" /etc/group; then - # No group with such gid exists - create group docker. + # No group with such gid exists - create group 'docker'. addgroup -g "${DOCKER_GID}" docker adduser docker docker else - # Group with such gid exists - add user "docker" to this group. + # Group with such gid exists - add user 'docker' to this group. DOCKER_GROUP_NAME=$(getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}') adduser docker "${DOCKER_GROUP_NAME}" fi else - # Docker socket belongs to "root" group - add user "docker" to this group. + # Docker socket belongs to 'root' group - add user 'docker' to this group. adduser docker root fi fi @@ -65,51 +66,41 @@ slugify() { make_image_cmd() { DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) - if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi - VOLUMES=$(echo "${1}" | jq -r 'select(.volumes != null) | .volumes | map(" -v " + .) | join("")') - PORTS=$(echo "${1}" | jq -r 'select(.ports != null) | .ports | map(" -p " + .) | join("")') - EXPOSE=$(echo "${1}" | jq -r 'select(.expose != null) | .expose | map(" --expose " + .) | join("")') + ENVIRONMENT=$(echo "${1}" | jq -r 'select(.environment != null) | .environment | map("--env " + .) | join(" ")') + EXPOSE=$(echo "${1}" | jq -r 'select(.expose != null) | .expose | map("--expose " + .) | join(" ")' ) NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') - NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network') - ENVIRONMENT=$(echo "${1}" | jq -r 'select(.environment != null) | .environment | map(" -e " + .) | join("")') - if [ -n "${NAME}" ]; then DOCKERARGS+=" --rm --name ${NAME} "; fi - if [ -n "${NETWORK}" ]; then DOCKERARGS+=" --network ${NETWORK} "; fi - if [ -n "${VOLUMES}" ]; then DOCKERARGS+="${VOLUMES}"; fi - if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS+="${ENVIRONMENT}"; fi - if [ -n "${PORTS}" ]; then DOCKERARGS+="${PORTS}"; fi - if [ -n "${EXPOSE}" ]; then DOCKERARGS+="${EXPOSE}"; fi + NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network | map("--network " + .) | join(" ")') + PORTS=$(echo "${1}" | jq -r 'select(.ports != null) | .ports | map("--publish " + .) | join(" ")') + VOLUMES=$(echo "${1}" | jq -r 'select(.volumes != null) | .volumes | map("--volume " + .) | join(" ")') + + if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi + DOCKERARGS+=" " + if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS+="${ENVIRONMENT} "; fi + if [ -n "${EXPOSE}" ]; then DOCKERARGS+="${EXPOSE} "; fi + if [ -n "${NAME}" ]; then DOCKERARGS+="--name ${NAME} "; fi + if [ -n "${NETWORK}" ]; then DOCKERARGS+="${NETWORK} "; fi + if [ -n "${PORTS}" ]; then DOCKERARGS+="${PORTS} "; fi + if [ -n "${VOLUMES}" ]; then DOCKERARGS+="${VOLUMES} "; fi + IMAGE=$(echo "${1}" | jq -r .image | envsubst) - TMP_COMMAND=$(echo "${1}" | jq -r .command) - echo "docker run ${DOCKERARGS} ${IMAGE} ${TMP_COMMAND}" + if [ "${IMAGE}" == "null" ]; then return; fi + + COMMAND=$(echo "${1}" | jq -r .command) + + echo "docker run ${DOCKERARGS} ${IMAGE} ${COMMAND}" } make_container_cmd() { DOCKERARGS=$(echo "${1}" | jq -r .dockerargs) if [ "${DOCKERARGS}" == "null" ]; then DOCKERARGS=; fi - SCRIPT_NAME=$(echo "${1}" | jq -r .name) - SCRIPT_NAME=$(slugify "${SCRIPT_NAME}") - PROJECT=$(echo "${1}" | jq -r .project) + CONTAINER=$(echo "${1}" | jq -r .container | envsubst) - TMP_COMMAND=$(echo "${1}" | jq -r .command) + if [ "${CONTAINER}" == "null" ]; then return; fi - if [ "${PROJECT}" != "null" ]; then - # Create bash script to detect all running containers. - if [ "${SCRIPT_NAME}" == "null" ]; then - SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) - fi -cat << EOF > "${HOME_DIR}"/projects/"${SCRIPT_NAME}".sh -#!/usr/bin/env bash -set -e + COMMAND=$(echo "${1}" | jq -r .command ) + if [ "${COMMAND}" == "null" ]; then return; fi -CONTAINERS=\$(docker ps --format '{{.Names}}' | grep -E "^${PROJECT}_${CONTAINER}.[0-9]+") -for CONTAINER_NAME in \${CONTAINERS}; do - docker exec "${DOCKERARGS} \${CONTAINER_NAME} ${TMP_COMMAND}" -done -EOF - echo "/bin/bash ${HOME_DIR}/projects/${SCRIPT_NAME}.sh" - else - echo "docker exec ${DOCKERARGS} ${CONTAINER} ${TMP_COMMAND}" - fi + echo "docker exec ${DOCKERARGS} ${CONTAINER} ${COMMAND}" } make_cmd() { @@ -176,84 +167,94 @@ function build_crontab() { ONSTART=() while read -r i ; do - SCHEDULE=$(jq -r .["$i"].schedule "${CONFIG}" | sed 's/\*/\\*/g') + KEY=$(jq -r .["$i"] "${CONFIG}") + + SCHEDULE=$(echo "${KEY}" | jq -r '.schedule' | sed 's/\*/\\*/g') if [ "${SCHEDULE}" == "null" ]; then - echo "'schedule' missing: $(jq -r .["$i"].schedule "${CONFIG}")" + echo "'schedule' missing: '${KEY}" continue fi SCHEDULE=$(parse_schedule "${SCHEDULE}" | sed 's/\\//g') - COMMAND=$(jq -r .["$i"].command "${CONFIG}") + COMMAND=$(echo "${KEY}" | jq -r '.command') if [ "${COMMAND}" == "null" ]; then - echo "'command' missing: '${COMMAND}'" + echo "'command' missing: '${KEY}'" continue fi - COMMENT=$(jq -r .["$i"].comment "${CONFIG}") - if [ "${COMMENT}" != "null" ]; then - COMMENT=" ${COMMENT}" - echo "#${COMMENT}" >> ${CRONTAB_FILE} - else - # Reset COMMENT to empty rather than keep the 'null' value. - COMMENT=" " - fi + COMMENT=$(echo "${KEY}" | jq -r '.comment') - SCRIPT_NAME=$(jq -r .["$i"].name "${CONFIG}") + SCRIPT_NAME=$(echo "${KEY}" | jq -r '.name') SCRIPT_NAME=$(slugify "${SCRIPT_NAME}") if [ "${SCRIPT_NAME}" == "null" ]; then SCRIPT_NAME=$(cat /proc/sys/kernel/random/uuid) fi - COMMAND="/bin/bash ${HOME_DIR}/jobs/${SCRIPT_NAME}.sh" -cat << EOF > "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh -#!/usr/bin/env bash -set -e + CRON_COMMAND=$(make_cmd "${KEY}") + + SCRIPT_PATH="${HOME_DIR}/jobs/${SCRIPT_NAME}.sh" + + touch "${SCRIPT_PATH}" + chmod +x "${SCRIPT_PATH}" -echo "start cron job **${SCRIPT_NAME}**${COMMENT}" -$(make_cmd "$(jq -c .["$i"] "${CONFIG}")") -EOF - TRIGGER=$(jq -r .["$i"].trigger "${CONFIG}") + { + echo "#\!/usr/bin/env bash" + echo "set -e" + echo "" + echo "echo \"start cron job __${SCRIPT_NAME}__\"" + echo "${CRON_COMMAND}" + } >> "${SCRIPT_PATH}" + + TRIGGER=$(echo "${KEY}" | jq -r '.trigger') if [ "${TRIGGER}" != "null" ]; then while read -r j ; do - TRIGGER_COMMAND=$(jq .["$i"].trigger["$j"].command "${CONFIG}") + TRIGGER_KEY=$(echo "${KEY}" | jq -r .trigger["$j"]) + + TRIGGER_COMMAND=$(echo "${TRIGGER_KEY}" | jq -r '.command') if [ "${TRIGGER_COMMAND}" == "null" ]; then - echo "'command' missing: '${TRIGGER_COMMAND}'" continue fi - make_cmd "${TRIGGER_COMMAND}" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh - done < <(jq -r '.['"$i"'].trigger|keys[]' "${CONFIG}") + + make_cmd "${TRIGGER_KEY}" >> "${SCRIPT_PATH}" + done < <(echo "${KEY}" | jq -r '.trigger | keys[]') fi - echo "echo \"end cron job **${SCRIPT_NAME}**${COMMENT}\"" >> "${HOME_DIR}"/jobs/"${SCRIPT_NAME}".sh + echo "echo \"end cron job __${SCRIPT_NAME}__\"" >> "${SCRIPT_PATH}" - echo "${SCHEDULE} ${COMMAND}" >> ${CRONTAB_FILE} + if [ "${COMMENT}" != "null" ]; then + echo "# ${COMMENT}" >> ${CRONTAB_FILE} + fi + echo "${SCHEDULE} ${SCRIPT_PATH}" >> ${CRONTAB_FILE} - if [ "$(jq -r .["$i"].onstart "${CONFIG}")" == "true" ]; then - ONSTART+=("${COMMAND}") + ONSTART_COMMAND=$(echo "${KEY}" | jq -r '.onstart') + if [ "${ONSTART_COMMAND}" == "true" ]; then + ONSTART+=("${SCRIPT_PATH}") fi - done < <(jq -r '.|keys[]' "${CONFIG}") + done < <(jq -r '. | keys[]' "${CONFIG}") - echo "##### crontab generation complete #####" + printf "##### crontab generated #####\n" cat ${CRONTAB_FILE} - echo "##### run commands with onstart #####" - for COMMAND in "${ONSTART[@]}"; do - echo "${COMMAND}" - ${COMMAND} & + printf "##### run commands with onstart #####\n" + for ONSTART_COMMAND in "${ONSTART[@]}"; do + printf "%s\n" "${ONSTART_COMMAND}" + ${ONSTART_COMMAND} & done + + printf "##### cron running #####\n" } start_app() { normalize_config export CONFIG=${HOME_DIR}/config.working.json if [ ! -f "${CONFIG}" ]; then - echo "generated ${CONFIG} missing. exiting." + printf "missing generated %s. exiting.\n" "${CONFIG}" exit 1 fi if [ "${1}" == "crond" ]; then build_crontab fi - echo "${@}" + printf "%s\n" "${@}" exec "${@}" } From 022399e16a5e023ce2c706d074752a46608766b2 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 4 Mar 2023 18:31:55 -0800 Subject: [PATCH 30/83] feat: adding TEST_MODE. --- Dockerfile | 1 + entrypoint.sh | 53 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b84154..899de74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN apk update && \ apk upgrade && \ apk add --no-cache \ bash \ + coreutils \ curl \ gettext \ jq \ diff --git a/entrypoint.sh b/entrypoint.sh index 37e8b87..90a3f33 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,8 +4,13 @@ set -e DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker +LOG_DIR=/var/log/crontab -if [ -z "${HOME_DIR}" ]; then +if [ -z "${HOME_DIR}" ] && [ -n "${TEST_MODE}" ]; then + HOME_DIR=/tmp/crontab-docker-testing + CRONTAB_FILE=${HOME_DIR}/test + LOG_DIR=${HOME_DIR}/crontab +elif [ -z "${HOME_DIR}" ]; then echo "HOME_DIR not set." exit 1 fi @@ -18,10 +23,9 @@ if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then fi if [ "${LOG_FILE}" == "" ]; then - LOG_DIR=/var/log/crontab LOG_FILE=${LOG_DIR}/jobs.log - mkdir -p ${LOG_DIR} - touch ${LOG_FILE} + mkdir -p "${LOG_DIR}" + touch "${LOG_FILE}" fi normalize_config() { @@ -115,10 +119,7 @@ make_cmd() { parse_schedule() { case $1 in - "@yearly") - echo "0 0 1 1 *" - ;; - "@annually") + "@yearly"|"@annually") echo "0 0 1 1 *" ;; "@monthly") @@ -156,6 +157,28 @@ parse_schedule() { echo "*/${TOTAL} * * * *" ;; + "@random") + for when in "$@" + do + if [ "$when" == "@random" ]; then + continue + fi + M H D="*" + case $when in + "@m") + M=$(shuf -i 0-6 -n 1) + ;; + "@h") + H=$(shuf -i 0-23 -n 1) + ;; + "@d") + D=$(shuf -i 0-6 -n 1) + ;; + esac + done + + echo "${M} ${H} * * ${D}" + ;; *) echo "${@}" ;; @@ -163,7 +186,7 @@ parse_schedule() { } function build_crontab() { - rm -rf ${CRONTAB_FILE} + rm -rf "${CRONTAB_FILE}" ONSTART=() while read -r i ; do @@ -203,7 +226,7 @@ function build_crontab() { echo "" echo "echo \"start cron job __${SCRIPT_NAME}__\"" echo "${CRON_COMMAND}" - } >> "${SCRIPT_PATH}" + } > "${SCRIPT_PATH}" TRIGGER=$(echo "${KEY}" | jq -r '.trigger') if [ "${TRIGGER}" != "null" ]; then @@ -222,9 +245,9 @@ function build_crontab() { echo "echo \"end cron job __${SCRIPT_NAME}__\"" >> "${SCRIPT_PATH}" if [ "${COMMENT}" != "null" ]; then - echo "# ${COMMENT}" >> ${CRONTAB_FILE} + echo "# ${COMMENT}" >> "${CRONTAB_FILE}" fi - echo "${SCHEDULE} ${SCRIPT_PATH}" >> ${CRONTAB_FILE} + echo "${SCHEDULE} ${SCRIPT_PATH}" >> "${CRONTAB_FILE}" ONSTART_COMMAND=$(echo "${KEY}" | jq -r '.onstart') if [ "${ONSTART_COMMAND}" == "true" ]; then @@ -233,7 +256,7 @@ function build_crontab() { done < <(jq -r '. | keys[]' "${CONFIG}") printf "##### crontab generated #####\n" - cat ${CRONTAB_FILE} + cat "${CRONTAB_FILE}" printf "##### run commands with onstart #####\n" for ONSTART_COMMAND in "${ONSTART[@]}"; do @@ -258,6 +281,8 @@ start_app() { exec "${@}" } -ensure_docker_socket_accessible +if [ -z "${TEST_MODE}" ]; then + ensure_docker_socket_accessible +fi printf "โœจ starting crontab container โœจ\n" start_app "${@}" From 2b364b93a7146b8c0007307cdd6bb2079c79ab7a Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 5 Mar 2023 12:02:18 -0800 Subject: [PATCH 31/83] fix: fixing @random since it was broken and removing @every since it never worked. --- entrypoint.sh | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 90a3f33..247943d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -118,8 +118,11 @@ make_cmd() { } parse_schedule() { - case $1 in - "@yearly"|"@annually") + IFS=" " + read -a params <<< "$@" + + case ${params[0]} in + "@yearly" | "@annually") echo "0 0 1 1 *" ;; "@monthly") @@ -137,36 +140,16 @@ parse_schedule() { "@hourly") echo "0 * * * *" ;; - "@every") - TIME=$2 - TOTAL=0 - - M=$(echo "${TIME}" | grep -o '[0-9]\+m') - H=$(echo "${TIME}" | grep -o '[0-9]\+h') - D=$(echo "${TIME}" | grep -o '[0-9]\+d') - - if [ -n "${M}" ]; then - TOTAL=$((TOTAL + ${M::-1})) - fi - if [ -n "${H}" ]; then - TOTAL=$((TOTAL + ${H::-1} * 60)) - fi - if [ -n "${D}" ]; then - TOTAL=$((TOTAL + ${D::-1} * 60 * 24)) - fi - - echo "*/${TOTAL} * * * *" - ;; "@random") - for when in "$@" + M="*" + H="*" + D="*" + + for when in "${params[@]:1}" do - if [ "$when" == "@random" ]; then - continue - fi - M H D="*" case $when in "@m") - M=$(shuf -i 0-6 -n 1) + M=$(shuf -i 0-59 -n 1) ;; "@h") H=$(shuf -i 0-23 -n 1) @@ -180,7 +163,7 @@ parse_schedule() { echo "${M} ${H} * * ${D}" ;; *) - echo "${@}" + echo "${params[@]}" ;; esac } From 899ec9c46ed7aabd2ace02ada9d4a3c3243ec95a Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 21 May 2023 11:43:46 -0700 Subject: [PATCH 32/83] fix: allow multiple networks. --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 247943d..3df3fde 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -73,7 +73,7 @@ make_image_cmd() { ENVIRONMENT=$(echo "${1}" | jq -r 'select(.environment != null) | .environment | map("--env " + .) | join(" ")') EXPOSE=$(echo "${1}" | jq -r 'select(.expose != null) | .expose | map("--expose " + .) | join(" ")' ) NAME=$(echo "${1}" | jq -r 'select(.name != null) | .name') - NETWORK=$(echo "${1}" | jq -r 'select(.network != null) | .network | map("--network " + .) | join(" ")') + NETWORKS=$(echo "${1}" | jq -r 'select(.networks != null) | .networks | map("--network " + .) | join(" ")') PORTS=$(echo "${1}" | jq -r 'select(.ports != null) | .ports | map("--publish " + .) | join(" ")') VOLUMES=$(echo "${1}" | jq -r 'select(.volumes != null) | .volumes | map("--volume " + .) | join(" ")') @@ -82,7 +82,7 @@ make_image_cmd() { if [ -n "${ENVIRONMENT}" ]; then DOCKERARGS+="${ENVIRONMENT} "; fi if [ -n "${EXPOSE}" ]; then DOCKERARGS+="${EXPOSE} "; fi if [ -n "${NAME}" ]; then DOCKERARGS+="--name ${NAME} "; fi - if [ -n "${NETWORK}" ]; then DOCKERARGS+="${NETWORK} "; fi + if [ -n "${NETWORKS}" ]; then DOCKERARGS+="${NETWORKS} "; fi if [ -n "${PORTS}" ]; then DOCKERARGS+="${PORTS} "; fi if [ -n "${VOLUMES}" ]; then DOCKERARGS+="${VOLUMES} "; fi From 2b46bb29493a9310b36e905d34cacee37c2179ed Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 21 May 2023 11:44:58 -0700 Subject: [PATCH 33/83] chore: remove FUNDING.yml --- .github/FUNDING.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 8e3a4a8..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: [willfarrell]# Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From ccaf6059e7a221b53bab13c5f1dabbd716f66572 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 24 Jun 2023 14:20:32 -0700 Subject: [PATCH 34/83] chore: remove forked build.yml. --- .github/workflows/build.yml | 44 ------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 7d524eb..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: build - -on: - push: - branches: - - main - tags: - - '*' - schedule: - - cron: '0 0 * * *' - -jobs: - multi: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - -# - name: Login to DockerHub -# uses: docker/login-action@v1 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_PASSWORD }} - -# - if: github.ref == 'refs/heads/main' -# name: Conditional(Set tag as `latest`) -# run: echo "tag=willfarrell/crontab:latest" >> $GITHUB_ENV - -# - if: startsWith(github.ref, 'refs/tags/') -# name: Conditional(Set tag as `{version}`) -# run: echo "tag=willfarrell/crontab:${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - -# - name: Build and push -# uses: docker/build-push-action@v2 -# with: -# context: . -# file: ./Dockerfile -# push: true -# tags: | -# ${{ env.tag }} From e3bfdebe5d97938ff245727b15614403e2ffa668 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 24 Jun 2023 14:31:13 -0700 Subject: [PATCH 35/83] chore: move build to the repo rather than the dockerfiles repo. --- .github/FUNDING.yml | 3 ++ .github/workflows/build.yml | 75 +++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 32 +++++++++++++ Dockerfile | 40 +++++++++++++---- README.md | 27 +++++++---- config-samples/config.sample.toml | 1 - 6 files changed, 159 insertions(+), 19 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/build.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..d2b5d24 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +github: [SimplicityGuy] +ko_fi: robertwlodarczyk +custom: [paypal.me/RWlodarczyk] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1a02c05 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,75 @@ +--- +name: crontab + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '0 1 * * 6' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.actor }}/crontab + +jobs: + build-crontab: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository. + uses: actions/checkout@v3 + with: + submodules: true + + - name: Log in to the GitHub Container Registry. + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker. + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=schedule,pattern={{date 'YYYYMMDD'}} + + - name: Set up QEMU. + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx. + uses: docker/setup-buildx-action@v2 + with: + platforms: linux/amd64, linux/arm64 + + - name: Build and push Docker image to GitHub Container Registry. + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64, linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: true + sbom: true + + - name: Send notification to Discord. + uses: sarisia/actions-status-discord@v1.12.0 + if: always() + with: + webhook: ${{ secrets.DISCORD_WEBHOOK }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..38e4936 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-yaml + - id: detect-aws-credentials + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.23.2 + hooks: + - id: check-github-workflows + + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.16 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-gfm + + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint diff --git a/Dockerfile b/Dockerfile index 899de74..91263d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,44 @@ -FROM alpine:latest as rq-build +#hadolint ignore=DL3007 +FROM alpine:latest as builder + +LABEL org.opencontainers.image.title="crontab builder" \ + org.opencontainers.image.description="crontab builder" \ + org.opencontainers.image.authors="robert@simplicityguy.com" \ + org.opencontainers.image.source="https://github.com/SimplicityGuy/alertmanager-discord/blob/main/Dockerfile" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.created="$(date +'%Y-%m-%d')" \ + org.opencontainers.image.base.name="docker.io/library/alpine" ENV RQ_VERSION=1.0.2 WORKDIR /usr/bin/rq/ -RUN apk update && \ - apk upgrade && \ - apk add --no-cache \ +#hadolint ignore=DL3018 +RUN apk update --quiet --no-cache && \ + apk upgrade --quiet --no-cache && \ + apk add --quiet --no-cache \ upx && \ - wget https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ + rm /var/cache/apk/* && \ + wget --quiet https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ tar -xvf rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ upx --brute rq +#hadolint ignore=DL3007 FROM docker:latest as release +LABEL org.opencontainers.image.title="crontab" \ + org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ + org.opencontainers.image.authors="robert@simplicityguy.com" \ + org.opencontainers.image.source="https://github.com/SimplicityGuy/docker-crontab/blob/main/Dockerfile" \ + org.opencontainers.image.licenses="MIT" \ + org.opencontainers.image.created="$(date +'%Y-%m-%d')" \ + org.opencontainers.image.base.name="docker.io/library/docker" + ENV HOME_DIR=/opt/crontab -RUN apk update && \ - apk upgrade && \ - apk add --no-cache \ +#hadolint ignore=DL3018 +RUN apk update --quiet --no-cache && \ + apk upgrade --quiet --no-cache && \ + apk add --quiet --no-cache \ bash \ coreutils \ curl \ @@ -25,11 +46,12 @@ RUN apk update && \ jq \ tini \ wget && \ + rm /var/cache/apk/* && \ mkdir -p ${HOME_DIR}/jobs && \ rm -rf /etc/periodic /etc/crontabs/root && \ adduser -S docker -D -COPY --from=rq-build /usr/bin/rq/rq /usr/local/bin +COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh / ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] diff --git a/README.md b/README.md index 5a625ca..7198946 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,16 @@ # crontab +![crontab](https://github.com/SimplicityGuy/adocker-crontab/actions/workflows/build.yml/badge.svg) ![License: MIT](https://img.shields.io/github/license/SimplicityGuy/docker-crontab) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) + A simple wrapper over `docker` to all complex cron job to be run in other containers. ## Why? -Yes, I'm aware of [mcuadros/ofelia](https://github.com/mcuadros/ofelia) (>250MB when this was created), it was the main inspiration for this project. + +Yes, I'm aware of [mcuadros/ofelia](https://github.com/mcuadros/ofelia) (>250MB when this was created), it was the main inspiration for this project. A great project, don't get me wrong. It was just missing certain key enterprise features I felt were required to support where docker is heading. ## Features + - Easy to read schedule syntax allowed. - Allows for comments, cause we all need friendly reminders of what `update_script.sh` actually does. - Start an image using `image`. @@ -15,6 +19,7 @@ A great project, don't get me wrong. It was just missing certain key enterprise - Ability to share settings between cron jobs using `~~shared-settings` as a key. ## Config file + The config file can be specified in any of `json`, `toml`, or `yaml`, and can be defined as either an array or mapping (top-level keys will be ignored; can be useful for organizing commands) - `name`: Human readable name that will be used as the job filename. Will be converted into a slug. Optional. @@ -24,7 +29,7 @@ The config file can be specified in any of `json`, `toml`, or `yaml`, and can be - `image`: Docker images name (ex `library/alpine:3.5`). Optional. - `container`: Full container name. Ignored if `image` is included. Optional. - `dockerargs`: Command line docker `run`/`exec` arguments for full control. Defaults to ` `. -- `trigger`: Array of docker-crontab subset objects. Sub-set includes: `image`, `container`, `command`, `dockerargs` +- `trigger`: Array of docker-crontab subset objects. Sub-set includes: `image`, `container`, `command`, `dockerargs` - `onstart`: Run the command on `crontab` container start, set to `true`. Optional, defaults to false. See [`config-samples`](config-samples) for examples. @@ -54,6 +59,7 @@ See [`config-samples`](config-samples) for examples. ## How to use ### Command Line + ```bash docker build -t crontab . docker run -d \ @@ -65,16 +71,18 @@ docker run -d \ ``` ### Use with docker-compose + 1. Figure out which network name used for your docker-compose containers - * use `docker network ls` to see existing networks - * if your `docker-compose.yml` is in `my_dir` directory, you probably has network `my_dir_default` - * otherwise [read the docker-compose docs](https://docs.docker.com/compose/networking/) -2. Add `dockerargs` to your docker-crontab `config.json` - * use `--network NETWORK_NAME` to connect new container into docker-compose network - * use `--name NAME` to use named container - * e.g. `"dockerargs": "--it"` + - use `docker network ls` to see existing networks + - if your `docker-compose.yml` is in `my_dir` directory, you probably has network `my_dir_default` + - otherwise [read the docker-compose docs](https://docs.docker.com/compose/networking/) +1. Add `dockerargs` to your docker-crontab `config.json` + - use `--network NETWORK_NAME` to connect new container into docker-compose network + - use `--name NAME` to use named container + - e.g. `"dockerargs": "--it"` ### Dockerfile + ```Dockerfile FROM registry.gitlab.com/simplicityguy/docker/crontab @@ -82,6 +90,7 @@ COPY config.json ${HOME_DIR}/ ``` ### Logrotate Dockerfile + ```Dockerfile FROM registry.gitlab.com/simplicityguy/docker/crontab diff --git a/config-samples/config.sample.toml b/config-samples/config.sample.toml index 5320c3f..b185048 100644 --- a/config-samples/config.sample.toml +++ b/config-samples/config.sample.toml @@ -47,4 +47,3 @@ onstart = true command = "sh -c '/etc/scripts/make_hpkp ${NGINX_DOMAIN} && /usr/sbin/nginx -t && /usr/sbin/nginx -s reload'" project = "conduit" container = "nginx" - From 4cfb3021b8e42c1e7ff93cbf6c2b0771d3e380ef Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 24 Jun 2023 14:34:09 -0700 Subject: [PATCH 36/83] fix: typo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7198946..eaea849 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # crontab -![crontab](https://github.com/SimplicityGuy/adocker-crontab/actions/workflows/build.yml/badge.svg) ![License: MIT](https://img.shields.io/github/license/SimplicityGuy/docker-crontab) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) +![crontab](https://github.com/SimplicityGuy/docker-crontab/actions/workflows/build.yml/badge.svg) ![License: MIT](https://img.shields.io/github/license/SimplicityGuy/docker-crontab) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) A simple wrapper over `docker` to all complex cron job to be run in other containers. From ac49cae228b86cd8b2f0c03c6c440ff6c3c63f65 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 24 Jun 2023 14:36:48 -0700 Subject: [PATCH 37/83] chore: remove arm64. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1a02c05..65cb38f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,13 +55,13 @@ jobs: - name: Set up Docker Buildx. uses: docker/setup-buildx-action@v2 with: - platforms: linux/amd64, linux/arm64 + platforms: linux/amd64 - name: Build and push Docker image to GitHub Container Registry. uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64, linux/arm64 + platforms: linux/amd64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From c7a835217c3f26cd30a794fa4145e618fb100265 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 24 Jun 2023 14:39:30 -0700 Subject: [PATCH 38/83] chore: update and upgrade require the cache. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91263d5..d6155b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,8 @@ ENV RQ_VERSION=1.0.2 WORKDIR /usr/bin/rq/ #hadolint ignore=DL3018 -RUN apk update --quiet --no-cache && \ - apk upgrade --quiet --no-cache && \ +RUN apk update --quiet && \ + apk upgrade --quiet && \ apk add --quiet --no-cache \ upx && \ rm /var/cache/apk/* && \ @@ -36,8 +36,8 @@ LABEL org.opencontainers.image.title="crontab" \ ENV HOME_DIR=/opt/crontab #hadolint ignore=DL3018 -RUN apk update --quiet --no-cache && \ - apk upgrade --quiet --no-cache && \ +RUN apk update --quiet && \ + apk upgrade --quiet && \ apk add --quiet --no-cache \ bash \ coreutils \ From c3a1963b9c86966e85a7cb2f3e4edd6b9f43f00e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 25 Jun 2023 19:07:53 -0700 Subject: [PATCH 39/83] feat: cleanup old images on the 15th of the month. --- .github/workflows/cleanup.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000..73053bf --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,28 @@ +--- +name: cleanup + +on: + schedule: + - cron: '0 0 15 * *' + +env: + IMAGE_NAME: ${{ github.actor }}/docker-crontab + +jobs: + cleanup-docker-crontab: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Delete Docker images older than a month. + uses: snok/container-retention-policy@v2 + with: + account-type: personal + cut-off: One month ago UTC + keep-at-least: 4 + skip-tags: latest + image-names: ${{ env.IMAGE_NAME }} + token: ${{ secrets.GHCR_TOKEN }} From 9c8b32cd22d17fca2df97b5e805ff759e86ccb38 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 25 Jun 2023 19:37:31 -0700 Subject: [PATCH 40/83] chore: add discord notification to cleanup. --- .github/workflows/cleanup.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 73053bf..1954a04 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -26,3 +26,13 @@ jobs: skip-tags: latest image-names: ${{ env.IMAGE_NAME }} token: ${{ secrets.GHCR_TOKEN }} + + - name: Send notification to Discord. + uses: sarisia/actions-status-discord@v1.12.0 + if: always() + with: + title: ${{ env.IMAGE_NAME }} + description: | + succeded cleanup : ${{ steps.cleanup-images.outputs.deleted }} + failed cleanup : ${{ steps.cleanup-images.outputs.failed }} + webhook: ${{ secrets.DISCORD_WEBHOOK }} From 5aeeb19efb570385b9c78dcb9c03770554da7843 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 25 Jun 2023 19:39:09 -0700 Subject: [PATCH 41/83] fix: missed id. --- .github/workflows/cleanup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 1954a04..e20376d 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -18,6 +18,7 @@ jobs: steps: - name: Delete Docker images older than a month. + id: cleanup-images uses: snok/container-retention-policy@v2 with: account-type: personal From a72e84783e37b22c7d5a9a4419620f2558be2d58 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Tue, 31 Dec 2024 10:47:59 -0800 Subject: [PATCH 42/83] chore: update pre-commit. --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38e4936..ec974ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0 hooks: - id: check-added-large-files - id: check-executables-have-shebangs @@ -15,18 +15,18 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.2 + rev: cb3c2be894b151dff143b1baf6acbd55f2b7faed # frozen: 0.30.0 hooks: - id: check-github-workflows - repo: https://github.com/executablebooks/mdformat - rev: 0.7.16 + rev: e20b1ac5acb8aba0b49d3a9109c6e6b58684ee83 # frozen: 0.7.21 hooks: - id: mdformat additional_dependencies: - mdformat-gfm - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta + rev: c3dc18df7a501f02a560a2cc7ba3c69a85ca01d3 # frozen: v2.13.1-beta hooks: - id: hadolint From 5172ed0b2112fbf1cbc9788aade9a7af39408ba3 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Tue, 31 Dec 2024 10:48:22 -0800 Subject: [PATCH 43/83] fix: use docker user. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d6155b2..3e658ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,8 @@ RUN apk update --quiet && \ rm -rf /etc/periodic /etc/crontabs/root && \ adduser -S docker -D +USER docker + COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh / From a87292bd6cdfd2a226d5164d2946866160ee0b6d Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Tue, 31 Dec 2024 10:53:17 -0800 Subject: [PATCH 44/83] chore: casing. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e658ba..b6becaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ #hadolint ignore=DL3007 -FROM alpine:latest as builder +FROM alpine:latest AS builder LABEL org.opencontainers.image.title="crontab builder" \ org.opencontainers.image.description="crontab builder" \ @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:latest as release +FROM docker:latest AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ From 1a540df103c2dcfc7c59001179b11268ed401c49 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 5 Jan 2025 10:46:14 -0800 Subject: [PATCH 45/83] fix: make /var/log/crontab writeable for everyone. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b6becaa..16fe7f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,8 +48,10 @@ RUN apk update --quiet && \ wget && \ rm /var/cache/apk/* && \ mkdir -p ${HOME_DIR}/jobs && \ + mkdir -p /var/log/crontab && \ rm -rf /etc/periodic /etc/crontabs/root && \ - adduser -S docker -D + adduser -S docker -D && \ + chmod u=rwx,go=rw /var/log/crontab USER docker From 18880befdb1c96b07439dfe26422c1427f9edb12 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 5 Jan 2025 10:52:44 -0800 Subject: [PATCH 46/83] fix: LOG_DIR is not used anywhere, remove it. --- Dockerfile | 4 +--- entrypoint.sh | 8 -------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16fe7f6..b6becaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,8 @@ RUN apk update --quiet && \ wget && \ rm /var/cache/apk/* && \ mkdir -p ${HOME_DIR}/jobs && \ - mkdir -p /var/log/crontab && \ rm -rf /etc/periodic /etc/crontabs/root && \ - adduser -S docker -D && \ - chmod u=rwx,go=rw /var/log/crontab + adduser -S docker -D USER docker diff --git a/entrypoint.sh b/entrypoint.sh index 3df3fde..bc3892f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,12 +4,10 @@ set -e DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker -LOG_DIR=/var/log/crontab if [ -z "${HOME_DIR}" ] && [ -n "${TEST_MODE}" ]; then HOME_DIR=/tmp/crontab-docker-testing CRONTAB_FILE=${HOME_DIR}/test - LOG_DIR=${HOME_DIR}/crontab elif [ -z "${HOME_DIR}" ]; then echo "HOME_DIR not set." exit 1 @@ -22,12 +20,6 @@ if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST="tcp://docker:2375" fi -if [ "${LOG_FILE}" == "" ]; then - LOG_FILE=${LOG_DIR}/jobs.log - mkdir -p "${LOG_DIR}" - touch "${LOG_FILE}" -fi - normalize_config() { JSON_CONFIG={} if [ -f "${HOME_DIR}/config.json" ]; then From 799ccc8edc529de1c3b8f6cc1bcb68ae297c2dfe Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 5 Jan 2025 10:58:49 -0800 Subject: [PATCH 47/83] fix: permissions for /opt/crontab. --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6becaa..e531c6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,9 +47,10 @@ RUN apk update --quiet && \ tini \ wget && \ rm /var/cache/apk/* && \ - mkdir -p ${HOME_DIR}/jobs && \ rm -rf /etc/periodic /etc/crontabs/root && \ - adduser -S docker -D + adduser -S docker -D && \ + mkdir -p ${HOME_DIR}/jobs && \ + chown -R docker:docker ${HOME_DIR} USER docker From 176cb2e29ce3028f5948dca597a9b7ea576785c5 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 5 Jan 2025 11:05:58 -0800 Subject: [PATCH 48/83] fix: move back to root user to debug permissions more completely. --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e531c6b..61b4b82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,11 +48,7 @@ RUN apk update --quiet && \ wget && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ - adduser -S docker -D && \ - mkdir -p ${HOME_DIR}/jobs && \ - chown -R docker:docker ${HOME_DIR} - -USER docker + mkdir -p ${HOME_DIR}/jobs COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh / From 7725e2f697ba8c5a3278d47fa660581550abf63c Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 11:57:27 -0800 Subject: [PATCH 49/83] fix: handle users at Dockerfile instead of in entrypoint script. --- entrypoint.sh | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index bc3892f..e8c1d0d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,6 @@ set -e -DOCKER_SOCK=/var/run/docker.sock CRONTAB_FILE=/etc/crontabs/docker if [ -z "${HOME_DIR}" ] && [ -n "${TEST_MODE}" ]; then @@ -35,27 +34,6 @@ normalize_config() { jq -S -r '."~~shared-settings" as $shared | del(."~~shared-settings") | to_entries | map_values(.value + { name: .key } + $shared)' <<< "${JSON_CONFIG}" > "${HOME_DIR}"/config.working.json } -ensure_docker_socket_accessible() { - if ! grep -q "^docker:" /etc/group; then - # Ensure 'docker' user has permissions for docker socket (without changing permissions). - DOCKER_GID=$(stat -c '%g' ${DOCKER_SOCK}) - if [ "${DOCKER_GID}" != "0" ]; then - if ! grep -qE "^[^:]+:[^:]+:${DOCKER_GID}:" /etc/group; then - # No group with such gid exists - create group 'docker'. - addgroup -g "${DOCKER_GID}" docker - adduser docker docker - else - # Group with such gid exists - add user 'docker' to this group. - DOCKER_GROUP_NAME=$(getent group "${DOCKER_GID}" | awk -F':' '{{ print $1 }}') - adduser docker "${DOCKER_GROUP_NAME}" - fi - else - # Docker socket belongs to 'root' group - add user 'docker' to this group. - adduser docker root - fi - fi -} - slugify() { echo "${@}" | iconv -t ascii | sed -r s/[~^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr '[:upper:]' '[:lower:]' } @@ -256,8 +234,5 @@ start_app() { exec "${@}" } -if [ -z "${TEST_MODE}" ]; then - ensure_docker_socket_accessible -fi printf "โœจ starting crontab container โœจ\n" start_app "${@}" From 165ea6a764b7721f2a88ee1dd6075f7434773af7 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 11:57:44 -0800 Subject: [PATCH 50/83] chore: update pre-commit. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec974ce..c982ed1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: cb3c2be894b151dff143b1baf6acbd55f2b7faed # frozen: 0.30.0 + rev: 62833a79b57fcd1bc372b136911a0edca60c3dcb # frozen: 0.31.0 hooks: - id: check-github-workflows From 495c98f4b84eccb805f29dc90b79bc792995cf45 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 12:29:22 -0800 Subject: [PATCH 51/83] fix: use rootless. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61b4b82..10fa1b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:latest AS release +FROM docker:dind-rootless AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ @@ -51,11 +51,11 @@ RUN apk update --quiet && \ mkdir -p ${HOME_DIR}/jobs COPY --from=builder /usr/bin/rq/rq /usr/local/bin -COPY entrypoint.sh / +COPY entrypoint.sh /opt -ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"] +ENTRYPOINT ["docker-entrypoint.sh", "/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "6", "-c", "/etc/crontabs"] +CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] From 8cd66168bd0da7a7ff99510603e944738464313e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 12:39:20 -0800 Subject: [PATCH 52/83] fix: using docker user and back to dind. --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 10fa1b0..d6ab389 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:dind-rootless AS release +FROM docker:latest AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ @@ -48,12 +48,16 @@ RUN apk update --quiet && \ wget && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ - mkdir -p ${HOME_DIR}/jobs + adduser -S docker -D && \ + mkdir -p ${HOME_DIR}/jobs && \ + chown -R docker:root ${HOME_DIR} + +USER docker COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt -ENTRYPOINT ["docker-entrypoint.sh", "/sbin/tini", "--", "/opt/entrypoint.sh"] +ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 From 903fb6059c82c8046663121d07e660b2f841968c Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 13:19:37 -0800 Subject: [PATCH 53/83] fix: trying out gosu. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6ab389..5fe821c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,8 @@ RUN apk update --quiet && \ jq \ tini \ wget && \ + apk add --quiet --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing \ + gosu && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ adduser -S docker -D && \ @@ -57,7 +59,7 @@ USER docker COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt -ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] +ENTRYPOINT ["/usr/bin/gosu", "docker:docker", "/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 From 7b249750e641286972f4ec7da9f95449e794b6a5 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 11 Jan 2025 13:23:09 -0800 Subject: [PATCH 54/83] fix: change group. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5fe821c..194e7a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ USER docker COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt -ENTRYPOINT ["/usr/bin/gosu", "docker:docker", "/sbin/tini", "--", "/opt/entrypoint.sh"] +ENTRYPOINT ["/usr/bin/gosu", "docker", "/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 From 0338bed0e4797963c381fbd03c6094dd58873cd8 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Fri, 17 Jan 2025 21:47:24 +0000 Subject: [PATCH 55/83] fix: Dockerfile to reduce vulnerabilities --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 194e7a1..024f55a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:latest AS release +FROM docker:27.5.0-dind-alpine3.21 AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ From a2c6492ce877593a934f3b578c024ebfe92fee00 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Thu, 19 Jun 2025 14:27:07 -0700 Subject: [PATCH 56/83] chore: update pre-commit. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c982ed1..b47c1b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,12 +15,12 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 62833a79b57fcd1bc372b136911a0edca60c3dcb # frozen: 0.31.0 + rev: 06e4cc849d03f3a59ca223a4046f4bb5bb2aba6d # frozen: 0.33.0 hooks: - id: check-github-workflows - repo: https://github.com/executablebooks/mdformat - rev: e20b1ac5acb8aba0b49d3a9109c6e6b58684ee83 # frozen: 0.7.21 + rev: ff29be1a1ba8029d9375882aa2c812b62112a593 # frozen: 0.7.22 hooks: - id: mdformat additional_dependencies: From 3a0192479483e5e9c288eb594cb9a0c8bcad1622 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Thu, 19 Jun 2025 14:27:57 -0700 Subject: [PATCH 57/83] fix: attempt to fix startup issues. --- Dockerfile | 17 +++++++++++------ README.md | 18 ++++++++++++++++++ docker-compose.yml | 8 +++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 194e7a1..a8a98fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,9 @@ LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.created="$(date +'%Y-%m-%d')" \ org.opencontainers.image.base.name="docker.io/library/docker" +# Build argument for docker group ID, default to 999 which is common +ARG DOCKER_GID=999 + ENV HOME_DIR=/opt/crontab #hadolint ignore=DL3018 @@ -45,21 +48,23 @@ RUN apk update --quiet && \ gettext \ jq \ tini \ - wget && \ - apk add --quiet --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing \ - gosu && \ + wget \ + shadow && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ - adduser -S docker -D && \ + # Create docker group with same GID as host docker group + addgroup -g ${DOCKER_GID} docker && \ + # Create docker user and add to docker group + adduser -S docker -D -G docker && \ mkdir -p ${HOME_DIR}/jobs && \ - chown -R docker:root ${HOME_DIR} + chown -R docker:docker ${HOME_DIR} USER docker COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt -ENTRYPOINT ["/usr/bin/gosu", "docker", "/sbin/tini", "--", "/opt/entrypoint.sh"] +ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 diff --git a/README.md b/README.md index eaea849..525c329 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,24 @@ See [`config-samples`](config-samples) for examples. ## How to use +### Docker Group ID Configuration + +This container needs to access the Docker socket to manage other containers. To do this, the `docker` user inside the container must have the same group ID (GID) as the `docker` group on the host system. + +By default, the Dockerfile uses GID 999, which is common for the `docker` group on many systems. If your host system uses a different GID, you need to specify it during the build: + +```bash +# Find your host's docker group ID +getent group docker | cut -d: -f3 +# Or alternatively +stat -c '%g' /var/run/docker.sock + +# Then build with the correct GID +docker build --build-arg DOCKER_GID= -t crontab . +``` + +If you encounter the error `failed switching to "docker": operation not permitted`, it means the GIDs don't match. Rebuild the image with the correct GID. + ### Command Line ```bash diff --git a/docker-compose.yml b/docker-compose.yml index a7cf566..c21629b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,13 @@ services: command: "sh -c 'while :; do sleep 1; done'" crontab: - build: . + build: + context: . + args: + # Set this to match your host's docker group ID + # You can find it with: getent group docker | cut -d: -f3 + # Or alternatively: stat -c '%g' /var/run/docker.sock + DOCKER_GID: 999 restart: always volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" From 23e4961eac5d5df63611d4762ad43eaf13090493 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 19 Jun 2025 21:32:38 +0000 Subject: [PATCH 58/83] fix: Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE321-EXPAT-9459843 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732740 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732740 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732743 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732743 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b995a4..72b78b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:27.5.0-dind-alpine3.21 AS release +FROM docker:27.5.1-dind-alpine3.21 AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ From 243d4a3b796481639d7a0c4bc8ab39ffd129dd4f Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Thu, 19 Jun 2025 14:39:21 -0700 Subject: [PATCH 59/83] fix: fix docker build. --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9b995a4..e430506 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,9 @@ ARG DOCKER_GID=999 ENV HOME_DIR=/opt/crontab +# Set shell with pipefail option to ensure pipe failures are caught +SHELL ["/bin/ash", "-o", "pipefail", "-c"] + #hadolint ignore=DL3018 RUN apk update --quiet && \ apk upgrade --quiet && \ @@ -52,8 +55,10 @@ RUN apk update --quiet && \ shadow && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ - # Create docker group with same GID as host docker group - addgroup -g ${DOCKER_GID} docker && \ + # Remove docker group if it exists + getent group docker > /dev/null && delgroup docker || true && \ + # Check if GID is in use, if so use a different one + (getent group | grep -q ":${DOCKER_GID}:" && addgroup docker || addgroup -g ${DOCKER_GID} docker) && \ # Create docker user and add to docker group adduser -S docker -D -G docker && \ mkdir -p ${HOME_DIR}/jobs && \ From fc275693e998fde1ed209c7438259a4b982590aa Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Thu, 19 Jun 2025 21:50:26 +0000 Subject: [PATCH 60/83] fix: Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE321-EXPAT-9459843 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732740 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732740 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732743 - https://snyk.io/vuln/SNYK-ALPINE321-OPENSSH-8732743 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dae7e6b..d6acf7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apk update --quiet && \ upx --brute rq #hadolint ignore=DL3007 -FROM docker:27.5.1-dind-alpine3.21 AS release +FROM docker:28.2.2-dind-alpine3.21 AS release LABEL org.opencontainers.image.title="crontab" \ org.opencontainers.image.description="A docker job scheduler (aka crontab for docker)." \ From 9052ef782718c7d81d6a306847ac8a27a456b5bc Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Thu, 19 Jun 2025 15:04:27 -0700 Subject: [PATCH 61/83] fix: add su-exec. --- Dockerfile | 1 + entrypoint.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6acf7e..cc33b96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,7 @@ RUN apk update --quiet && \ curl \ gettext \ jq \ + su-exec \ tini \ wget \ shadow && \ diff --git a/entrypoint.sh b/entrypoint.sh index e8c1d0d..8d9a39e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,6 +14,8 @@ fi # Ensure dir exist - in case of volume mapping. mkdir -p "${HOME_DIR}"/jobs +# Ensure proper permissions for volume-mounted directories +chown -R docker:docker "${HOME_DIR}" if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST="tcp://docker:2375" @@ -231,7 +233,13 @@ start_app() { build_crontab fi printf "%s\n" "${@}" - exec "${@}" + + # Run the command as the docker user + if [ "$(id -u)" = "0" ]; then + exec su-exec docker "${@}" + else + exec "${@}" + fi } printf "โœจ starting crontab container โœจ\n" From 8fb02226e73e3224469e2acc06eb9e4ee0044e4e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 29 Jun 2025 13:05:11 -0700 Subject: [PATCH 62/83] chore: update claude docs. --- CLAUDE.md | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..1ac1cf8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,151 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is `docker-crontab`, a Docker-based cron job scheduler that allows running complex cron jobs in other containers. It's a lightweight alternative to mcuadros/ofelia with enterprise features. + +## Key Architecture + +### Core Components + +- **Dockerfile**: Multi-stage build using Alpine Linux base with Docker-in-Docker capability + + - Builder stage: Downloads and compresses `rq` tool for config parsing + - Release stage: Based on `docker:dind-alpine` with cron and Docker client + - Uses `su-exec` for proper user privilege handling + - Configurable Docker group ID via `DOCKER_GID` build arg (default: 999) + +- **entrypoint.sh**: Main orchestration script that: + + - Normalizes config files (JSON/YAML/TOML) using `rq` and `jq` + - Processes shared settings via `~~shared-settings` key + - Generates crontab entries and executable scripts + - Supports both `image` (docker run) and `container` (docker exec) execution modes + - Handles trigger chains and onstart commands + +### Configuration System + +- Supports JSON, YAML, and TOML config formats +- Config can be array or mapping (top-level keys ignored for organization) +- Special `~~shared-settings` key for shared configuration +- Key fields: `schedule`, `command`, `image`/`container`, `dockerargs`, `trigger`, `onstart` +- Schedule supports standard crontab syntax plus shortcuts (@hourly, @daily, @every 2m, etc.) +- Additional fields: `comment`, `name`, `environment`, `expose`, `networks`, `ports`, `volumes` + +### Job Execution Flow + +1. Config normalization: All formats converted to working JSON +1. Script generation: Each job becomes executable shell script in `/opt/crontab/jobs/` +1. Crontab creation: Standard crontab file generated with proper scheduling +1. Trigger processing: Post-job triggers executed in sequence +1. Onstart handling: Jobs marked with `onstart: true` run immediately + +## Development Commands + +### Building + +```bash +# Basic build +docker build -t crontab . + +# Build with custom Docker group ID +docker build --build-arg DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) -t crontab . +``` + +### Running + +```bash +# Command line execution +docker run -d \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v ./config-samples/config.sample.json:/opt/crontab/config.json:ro \ + -v ./logs:/var/log/crontab:rw \ + crontab + +# With host directory for persistent config/logs +# Container will create directories with proper permissions +docker run -d \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v $PWD/crontab-config:/opt/crontab:rw \ + -v $PWD/crontab-logs:/var/log/crontab:rw \ + crontab + +# Docker Compose +docker-compose up +``` + +### Testing + +```bash +# Test with sample configuration +docker run -d \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v ./config-samples/config.sample.json:/opt/crontab/config.json:ro \ + crontab + +# Debug mode - view generated crontab and scripts +docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v ./config-samples/config.sample.json:/opt/crontab/config.json:ro \ + -e TEST_MODE=1 \ + crontab bash -c "cat /tmp/crontab-docker-testing/test && ls -la /tmp/crontab-docker-testing/jobs/" +``` + +The repository includes sample configurations in `config-samples/` for testing different scenarios. + +## Important Configuration Notes + +- **Docker Socket Access**: Container requires read-only access to `/var/run/docker.sock` +- **User Permissions**: Uses `docker` user with configurable GID to match host Docker group +- **Volume Mounts**: Config and log directories should be mounted as volumes +- **Network Access**: For docker-compose usage, containers need network connectivity via `--network` in `dockerargs` + +## Troubleshooting + +### Common Issues + +- **"failed switching to 'docker': operation not permitted"**: Docker group GID mismatch + + - Solution: Rebuild with correct GID using `--build-arg DOCKER_GID=$(stat -c '%g' /var/run/docker.sock)` + +- **"Permission denied" creating directories**: Volume mount permissions issue + + - Solution: Ensure host directories have correct ownership before mounting + - Quick fix: `sudo chown -R $(id -u):$(getent group docker | cut -d: -f3) /path/to/host/directory` + - Or let container create directories (it runs as root initially, then drops privileges) + +- **Jobs not executing**: Check crontab generation and script permissions + + - Debug: Use `TEST_MODE=1` environment variable to inspect generated files + +- **Container networking issues**: Ensure proper network configuration in `dockerargs` + + - For docker-compose: Add `--network ` to dockerargs + +### File Locations + +- Generated scripts: `/opt/crontab/jobs/` +- Working config: `/opt/crontab/config.working.json` +- Crontab file: `/etc/crontabs/docker` +- Logs: Container stdout/stderr (configure external logging as needed) + +## Security Considerations + +- Container runs as non-root `docker` user for security +- Docker socket access is read-only to prevent container escape +- Uses `su-exec` for privilege dropping instead of `sudo` +- Multi-stage build minimizes attack surface +- SBOM and provenance generation enabled in CI/CD + +## CI/CD + +GitHub Actions workflow (`.github/workflows/build.yml`): + +- Builds on push to main and PRs +- Multi-platform support (linux/amd64) +- Publishes to GitHub Container Registry (`ghcr.io`) +- Includes security scanning with SBOM and provenance +- Discord notifications for build status +- Weekly scheduled builds for base image security updates From 48f67b1e2639ac1c8a07e9ac5ffc1e177907c10e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 29 Jun 2025 13:05:34 -0700 Subject: [PATCH 63/83] chore: update pre-commit. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b47c1b8..136dc0f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 06e4cc849d03f3a59ca223a4046f4bb5bb2aba6d # frozen: 0.33.0 + rev: a1419a25b3ec6c91a963e044f03f6dc197930b10 # frozen: 0.33.1 hooks: - id: check-github-workflows From 60bc7f4bbb8870497c75cefaec96f93c3f561da0 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 29 Jun 2025 13:05:57 -0700 Subject: [PATCH 64/83] fix: permission error. --- Dockerfile | 4 ++-- entrypoint.sh | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index cc33b96..c22df95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,11 +65,11 @@ RUN apk update --quiet && \ mkdir -p ${HOME_DIR}/jobs && \ chown -R docker:docker ${HOME_DIR} -USER docker - COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt +# Start as root to handle volume permissions, then drop to docker user in entrypoint + ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ diff --git a/entrypoint.sh b/entrypoint.sh index 8d9a39e..2ffda2c 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,9 +13,17 @@ elif [ -z "${HOME_DIR}" ]; then fi # Ensure dir exist - in case of volume mapping. -mkdir -p "${HOME_DIR}"/jobs -# Ensure proper permissions for volume-mounted directories -chown -R docker:docker "${HOME_DIR}" +# This needs to run as root to set proper permissions +if [ "$(id -u)" = "0" ]; then + mkdir -p "${HOME_DIR}"/jobs + chown -R docker:docker "${HOME_DIR}" +else + # If not root, try to create directory (may fail if permissions are wrong) + mkdir -p "${HOME_DIR}"/jobs 2>/dev/null || { + echo "Warning: Cannot create ${HOME_DIR}/jobs directory. Ensure proper volume permissions." + echo "Run: sudo chown -R $(id -u docker):$(id -g docker) /path/to/host/directory" + } +fi if [ -z "${DOCKER_HOST}" ] && [ -a "${DOCKER_PORT_2375_TCP}" ]; then export DOCKER_HOST="tcp://docker:2375" From 598be37986352b638b1a445bb1c5a9ee52d1d2fd Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Mon, 30 Jun 2025 13:55:47 -0700 Subject: [PATCH 65/83] fix: fix the error 'can't set groups: Operation not permitted'. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c22df95..f94acf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] +CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs", "-S"] From 05710ab9256ef91e64a2f972c64e32c2263cb963 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 6 Jul 2025 14:25:08 -0700 Subject: [PATCH 66/83] fix: use crond -s flag to prevent user switching errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace -S flag with -s flag to run crond in single-user mode. This prevents "can't set groups: Operation not permitted" errors since su-exec already handles the user switching to docker user. ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f94acf2..77c4add 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs", "-S"] +CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs", "-s"] From c5a802ce5f790ff0eb67d58ca0c3ce67488e03ef Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 19 Jul 2025 14:51:35 -0700 Subject: [PATCH 67/83] fix: correct scripts and update dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix shellcheck warning in entrypoint.sh (add -r flag to read command) - Fix incorrect source URL in Dockerfile (was pointing to alertmanager-discord) - Update docker-compose.yml to version 3.8 and change config volume to read-only - Update GitHub Actions to use latest action versions for better security and features - Remove invalid -s flag from crond CMD (already fixed in previous commit) ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/build.yml | 12 ++++++------ Dockerfile | 4 ++-- docker-compose.yml | 4 ++-- entrypoint.sh | 9 ++++++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65cb38f..a063b4b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,13 +26,13 @@ jobs: steps: - name: Checkout repository. - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Log in to the GitHub Container Registry. if: github.event_name != 'pull_request' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -40,7 +40,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker. id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -50,15 +50,15 @@ jobs: type=schedule,pattern={{date 'YYYYMMDD'}} - name: Set up QEMU. - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx. - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: platforms: linux/amd64 - name: Build and push Docker image to GitHub Container Registry. - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64 diff --git a/Dockerfile b/Dockerfile index 77c4add..b39254e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM alpine:latest AS builder LABEL org.opencontainers.image.title="crontab builder" \ org.opencontainers.image.description="crontab builder" \ org.opencontainers.image.authors="robert@simplicityguy.com" \ - org.opencontainers.image.source="https://github.com/SimplicityGuy/alertmanager-discord/blob/main/Dockerfile" \ + org.opencontainers.image.source="https://github.com/SimplicityGuy/docker-crontab/blob/main/Dockerfile" \ org.opencontainers.image.licenses="MIT" \ org.opencontainers.image.created="$(date +'%Y-%m-%d')" \ org.opencontainers.image.base.name="docker.io/library/alpine" @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs", "-s"] +CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] diff --git a/docker-compose.yml b/docker-compose.yml index c21629b..3fd9dd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2.1" +version: "3.8" services: myapp: @@ -17,4 +17,4 @@ services: restart: always volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "${PWD}/config-samples/config.sample.mapping.json:/opt/crontab/config.json:rw" + - "${PWD}/config-samples/config.sample.mapping.json:/opt/crontab/config.json:ro" diff --git a/entrypoint.sh b/entrypoint.sh index 2ffda2c..37564e5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -99,7 +99,7 @@ make_cmd() { parse_schedule() { IFS=" " - read -a params <<< "$@" + read -r -a params <<< "$@" case ${params[0]} in "@yearly" | "@annually") @@ -242,8 +242,11 @@ start_app() { fi printf "%s\n" "${@}" - # Run the command as the docker user - if [ "$(id -u)" = "0" ]; then + # Run crond as root so it can switch users for cron jobs + # Other commands run as docker user for security + if [ "${1}" == "crond" ]; then + exec "${@}" + elif [ "$(id -u)" = "0" ]; then exec su-exec docker "${@}" else exec "${@}" From c637f743a0485b1e579e665f8ef12a0b37a913c0 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sat, 26 Jul 2025 11:12:03 -0700 Subject: [PATCH 68/83] fix: filter out unsupported -s flag for BusyBox crond MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BusyBox v1.37.0 crond doesn't support the -s flag that was attempted in previous versions. This fix adds argument filtering in entrypoint.sh to gracefully handle and remove any -s flags passed to crond, preventing the "unrecognized option: s" error. The solution maintains all previous fixes including proper user switching via su-exec and running crond as root for cron job execution. ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- entrypoint.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 37564e5..09d1710 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -240,16 +240,37 @@ start_app() { if [ "${1}" == "crond" ]; then build_crontab fi - printf "%s\n" "${@}" + + # Filter out invalid crond flags + # BusyBox crond doesn't support -s flag + local filtered_args=() + local skip_next=false + + for arg in "$@"; do + if [ "$skip_next" = true ]; then + skip_next=false + continue + fi + + # Skip -s flag if it appears (was used in previous versions but not supported by BusyBox) + if [ "$arg" = "-s" ]; then + echo "Warning: Skipping unsupported -s flag for BusyBox crond" + continue + fi + + filtered_args+=("$arg") + done + + printf "%s\n" "${filtered_args[@]}" # Run crond as root so it can switch users for cron jobs # Other commands run as docker user for security if [ "${1}" == "crond" ]; then - exec "${@}" + exec "${filtered_args[@]}" elif [ "$(id -u)" = "0" ]; then - exec su-exec docker "${@}" + exec su-exec docker "${filtered_args[@]}" else - exec "${@}" + exec "${filtered_args[@]}" fi } From 26f559135116e18bf075dbae5c1f2286779b764b Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Sun, 3 Aug 2025 13:52:45 -0700 Subject: [PATCH 69/83] fix: moving the CRONTAB_FILE. --- Dockerfile | 2 +- entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b39254e..35fd80f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] +CMD ["crond", "-f", "-d", "7", "-c", "${HOME_DIR}"] diff --git a/entrypoint.sh b/entrypoint.sh index 09d1710..231fd4b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,7 @@ set -e -CRONTAB_FILE=/etc/crontabs/docker +CRONTAB_FILE="${HOME_DIR}"/crontab if [ -z "${HOME_DIR}" ] && [ -n "${TEST_MODE}" ]; then HOME_DIR=/tmp/crontab-docker-testing From 6d8f92f4d6da71b5a52d5af8545facd1f00eaf0d Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Fri, 15 Aug 2025 18:09:47 -0700 Subject: [PATCH 70/83] fix: Issue was that the environment variable wasn't being expanded in the CMD instruction. Docker doesn't perform shell variable expansion in the exec form of CMD. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 35fd80f..01c3470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "${HOME_DIR}"] +CMD ["crond", "-f", "-d", "7", "-c", "/opt/crontab"] From 26afa4f24d6c089dc7afb73b42000605b7db397e Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Fri, 15 Aug 2025 18:10:47 -0700 Subject: [PATCH 71/83] chore: update pre-commit. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 136dc0f..9d04c92 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-added-large-files - id: check-executables-have-shebangs @@ -15,7 +15,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: a1419a25b3ec6c91a963e044f03f6dc197930b10 # frozen: 0.33.1 + rev: 54da05914997e6b04e4db33ed6757d744984c68b # frozen: 0.33.2 hooks: - id: check-github-workflows From 1d282551c7ad86b633c9fb29529a2996765007f2 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Mon, 18 Aug 2025 21:11:33 -0700 Subject: [PATCH 72/83] fix: BusyBox crond crontab directory configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue was that BusyBox crond was trying to interpret subdirectories in /opt/crontab as usernames. This fix: - Installs the generated crontab to /etc/crontabs/docker where BusyBox expects user crontab files - Updates the crond command to use /etc/crontabs instead of /opt/crontab - Ensures proper permissions (600) on the crontab file This resolves the "no such user" errors for 'crontab' and 'jobs' directories. ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- Dockerfile | 2 +- entrypoint.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 01c3470..b39254e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,4 +75,4 @@ ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/opt/crontab"] +CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] diff --git a/entrypoint.sh b/entrypoint.sh index 231fd4b..dfdfef5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -221,6 +221,12 @@ function build_crontab() { printf "##### crontab generated #####\n" cat "${CRONTAB_FILE}" + # Install the crontab for the docker user + # BusyBox crond expects crontab files in /etc/crontabs/ + mkdir -p /etc/crontabs + cp "${CRONTAB_FILE}" /etc/crontabs/docker + chmod 600 /etc/crontabs/docker + printf "##### run commands with onstart #####\n" for ONSTART_COMMAND in "${ONSTART[@]}"; do printf "%s\n" "${ONSTART_COMMAND}" From 9c42e534a63549ce56f7295c45f07f62122aae01 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Tue, 30 Dec 2025 15:50:19 -0800 Subject: [PATCH 73/83] chore: update deps --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d04c92..6c8c303 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,18 +15,18 @@ repos: - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 54da05914997e6b04e4db33ed6757d744984c68b # frozen: 0.33.2 + rev: b035497fb64e3f9faa91e833331688cc185891e6 # frozen: 0.36.0 hooks: - id: check-github-workflows - repo: https://github.com/executablebooks/mdformat - rev: ff29be1a1ba8029d9375882aa2c812b62112a593 # frozen: 0.7.22 + rev: 2d496dbc18e31b83a1596685347ffe0b6041daf0 # frozen: 1.0.0 hooks: - id: mdformat additional_dependencies: - mdformat-gfm - repo: https://github.com/hadolint/hadolint - rev: c3dc18df7a501f02a560a2cc7ba3c69a85ca01d3 # frozen: v2.13.1-beta + rev: 4e697ba704fd23b2409b947a319c19c3ee54d24f # frozen: v2.14.0 hooks: - id: hadolint From 6ab1073f8b7a434b1d050c8543ba099495af5115 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Tue, 30 Dec 2025 16:31:54 -0800 Subject: [PATCH 74/83] fix: use user-writable crontab directory for non-root execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves permission error "cannot create regular file '/etc/crontabs/docker'" by moving crontab files to /opt/crontab/crontabs/ owned by docker user. Changes: - Modified entrypoint.sh to write crontab to /opt/crontab/crontabs/docker - Updated Dockerfile to create crontabs directory with docker:docker ownership - Added SUID bit to /usr/bin/crontab for proper crontab management - Changed CMD to use custom crontabs directory (-c /opt/crontab/crontabs) - Removed root requirement for crond execution (now runs as docker user) - Enhanced documentation with security model and troubleshooting sections - Updated all file path references in CLAUDE.md and README.md Security improvements: - Container now runs crond as non-root docker user - Privilege separation via su-exec (starts as root, drops to docker user) - Better handling of read-only volume mounts ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- CLAUDE.md | 11 ++++-- Dockerfile | 12 ++++--- README.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++++-- entrypoint.sh | 35 ++++++++++--------- 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1ac1cf8..ab6e48d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,8 +22,10 @@ This is `docker-crontab`, a Docker-based cron job scheduler that allows running - Normalizes config files (JSON/YAML/TOML) using `rq` and `jq` - Processes shared settings via `~~shared-settings` key - Generates crontab entries and executable scripts + - Installs crontab files in user-writable directory (`/opt/crontab/crontabs`) - Supports both `image` (docker run) and `container` (docker exec) execution modes - Handles trigger chains and onstart commands + - Drops privileges to `docker` user for security ### Configuration System @@ -113,12 +115,14 @@ The repository includes sample configurations in `config-samples/` for testing d - **"Permission denied" creating directories**: Volume mount permissions issue - Solution: Ensure host directories have correct ownership before mounting - - Quick fix: `sudo chown -R $(id -u):$(getent group docker | cut -d: -f3) /path/to/host/directory` - - Or let container create directories (it runs as root initially, then drops privileges) + - Quick fix: `sudo chown -R $(id -u):$(id -g) /path/to/host/directory` + - Or let container create directories (it runs as root initially, then drops privileges to `docker` user) - **Jobs not executing**: Check crontab generation and script permissions - Debug: Use `TEST_MODE=1` environment variable to inspect generated files + - Verify crontab file exists: Check `/opt/crontab/crontabs/docker` inside container + - Check logs: Container outputs cron job execution to stdout/stderr - **Container networking issues**: Ensure proper network configuration in `dockerargs` @@ -128,7 +132,8 @@ The repository includes sample configurations in `config-samples/` for testing d - Generated scripts: `/opt/crontab/jobs/` - Working config: `/opt/crontab/config.working.json` -- Crontab file: `/etc/crontabs/docker` +- Crontab directory: `/opt/crontab/crontabs/` +- Crontab file: `/opt/crontab/crontabs/docker` - Logs: Container stdout/stderr (configure external logging as needed) ## Security Considerations diff --git a/Dockerfile b/Dockerfile index b39254e..bf9e4a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,23 +56,27 @@ RUN apk update --quiet && \ shadow && \ rm /var/cache/apk/* && \ rm -rf /etc/periodic /etc/crontabs/root && \ + # Set SUID on crontab command so it can modify crontab files + chmod u+s /usr/bin/crontab && \ # Remove docker group if it exists getent group docker > /dev/null && delgroup docker || true && \ # Check if GID is in use, if so use a different one (getent group | grep -q ":${DOCKER_GID}:" && addgroup docker || addgroup -g ${DOCKER_GID} docker) && \ # Create docker user and add to docker group adduser -S docker -D -G docker && \ - mkdir -p ${HOME_DIR}/jobs && \ + mkdir -p ${HOME_DIR}/jobs ${HOME_DIR}/crontabs && \ chown -R docker:docker ${HOME_DIR} COPY --from=builder /usr/bin/rq/rq /usr/local/bin COPY entrypoint.sh /opt -# Start as root to handle volume permissions, then drop to docker user in entrypoint - ENTRYPOINT ["/sbin/tini", "--", "/opt/entrypoint.sh"] HEALTHCHECK --interval=5s --timeout=3s \ CMD ps aux | grep '[c]rond' || exit 1 -CMD ["crond", "-f", "-d", "7", "-c", "/etc/crontabs"] +# Run crond with custom crontabs directory owned by docker user +# -f: foreground mode +# -d 7: debug level 7 (highest) +# -c: crontabs directory +CMD ["crond", "-f", "-d", "7", "-c", "/opt/crontab/crontabs"] diff --git a/README.md b/README.md index 525c329..e7b8983 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,27 @@ See [`config-samples`](config-samples) for examples. } ``` +## Architecture & Security + +### Security Model + +The container is designed with security best practices: + +- **Non-root execution**: Container runs as the `docker` user (not root) for security +- **Privilege separation**: Starts as root to set up directories, then drops to `docker` user via `su-exec` +- **Read-only Docker socket**: Docker socket is mounted read-only to prevent container escape +- **User-writable directories**: Crontab and job files stored in `/opt/crontab/` owned by `docker` user +- **SUID crontab**: The `crontab` command has SUID bit set for proper crontab file management + +### Directory Structure + +- `/opt/crontab/` - Main working directory (can be volume mounted) + - `config.json` (or `.yaml`, `.toml`) - Your configuration file + - `config.working.json` - Normalized configuration (auto-generated) + - `jobs/` - Generated shell scripts for each cron job + - `crontabs/` - Crontab files for BusyBox crond + - `docker` - Crontab file for the `docker` user + ## How to use ### Docker Group ID Configuration @@ -109,12 +130,78 @@ COPY config.json ${HOME_DIR}/ ### Logrotate Dockerfile +This example shows how to extend the crontab image for custom use cases: + ```Dockerfile -FROM registry.gitlab.com/simplicityguy/docker/crontab +FROM ghcr.io/simplicityguy/crontab RUN apk add --no-cache logrotate -RUN echo "*/5 * * * * /usr/sbin/logrotate /etc/logrotate.conf" >> /etc/crontabs/logrotate COPY logrotate.conf /etc/logrotate.conf +# Use the config.json approach instead of manually editing crontab files +COPY config.json ${HOME_DIR}/ +``` + +## Troubleshooting + +### Permission Errors + +**Issue**: `failed switching to 'docker': operation not permitted` + +**Cause**: Docker group GID mismatch between host and container. + +**Solution**: Rebuild the image with the correct Docker group ID: + +```bash +# Find your host's docker group ID +stat -c '%g' /var/run/docker.sock -CMD ["crond", "-f"] +# Rebuild with the correct GID +docker build --build-arg DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) -t crontab . ``` + +### Jobs Not Executing + +**Issue**: Cron jobs defined in config but not running. + +**Troubleshooting steps**: + +1. Check if crontab file was generated: + + ```bash + docker exec cat /opt/crontab/crontabs/docker + ``` + +1. Verify job scripts exist: + + ```bash + docker exec ls -la /opt/crontab/jobs/ + ``` + +1. Check crond is running: + + ```bash + docker exec ps aux | grep crond + ``` + +1. View container logs for cron execution output: + + ```bash + docker logs + ``` + +### Directory Permission Issues + +**Issue**: Container can't create directories when using volume mounts. + +**Solution**: Ensure the host directory has correct permissions before mounting: + +```bash +# Create directory and set ownership +mkdir -p /path/to/crontab +chown -R $(id -u):$(id -g) /path/to/crontab + +# Then run container with volume mount +docker run -v /path/to/crontab:/opt/crontab:rw ... +``` + +Alternatively, let the container create the directories on first run (it starts as root, creates directories, then drops to `docker` user). diff --git a/entrypoint.sh b/entrypoint.sh index dfdfef5..6bed16b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -15,12 +15,15 @@ fi # Ensure dir exist - in case of volume mapping. # This needs to run as root to set proper permissions if [ "$(id -u)" = "0" ]; then - mkdir -p "${HOME_DIR}"/jobs - chown -R docker:docker "${HOME_DIR}" + mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/crontabs + # Only chown the directories we create, not the entire HOME_DIR (to avoid issues with read-only mounts) + chown docker:docker "${HOME_DIR}"/jobs "${HOME_DIR}"/crontabs 2>/dev/null || true + # Try to chown HOME_DIR itself, but ignore errors for read-only mounts + chown docker:docker "${HOME_DIR}" 2>/dev/null || true else - # If not root, try to create directory (may fail if permissions are wrong) - mkdir -p "${HOME_DIR}"/jobs 2>/dev/null || { - echo "Warning: Cannot create ${HOME_DIR}/jobs directory. Ensure proper volume permissions." + # If not root, try to create directories (may fail if permissions are wrong) + mkdir -p "${HOME_DIR}"/jobs "${HOME_DIR}"/crontabs 2>/dev/null || { + echo "Warning: Cannot create ${HOME_DIR} directories. Ensure proper volume permissions." echo "Run: sudo chown -R $(id -u docker):$(id -g docker) /path/to/host/directory" } fi @@ -221,11 +224,16 @@ function build_crontab() { printf "##### crontab generated #####\n" cat "${CRONTAB_FILE}" - # Install the crontab for the docker user - # BusyBox crond expects crontab files in /etc/crontabs/ - mkdir -p /etc/crontabs - cp "${CRONTAB_FILE}" /etc/crontabs/docker - chmod 600 /etc/crontabs/docker + # Copy crontab file to a directory owned by docker user + # BusyBox crond expects files in the crontabs directory to be named after the user + CRONTABS_DIR="${HOME_DIR}/crontabs" + mkdir -p "${CRONTABS_DIR}" + cp "${CRONTAB_FILE}" "${CRONTABS_DIR}/docker" + chmod 600 "${CRONTABS_DIR}/docker" + # Ensure ownership is correct + if [ "$(id -u)" = "0" ]; then + chown docker:docker "${CRONTABS_DIR}" "${CRONTABS_DIR}/docker" + fi printf "##### run commands with onstart #####\n" for ONSTART_COMMAND in "${ONSTART[@]}"; do @@ -269,11 +277,8 @@ start_app() { printf "%s\n" "${filtered_args[@]}" - # Run crond as root so it can switch users for cron jobs - # Other commands run as docker user for security - if [ "${1}" == "crond" ]; then - exec "${filtered_args[@]}" - elif [ "$(id -u)" = "0" ]; then + # Run as docker user for security + if [ "$(id -u)" = "0" ]; then exec su-exec docker "${filtered_args[@]}" else exec "${filtered_args[@]}" From a25fe1bfa91310f224b4360bb51e211442a463b9 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 11:55:12 -0800 Subject: [PATCH 75/83] chore(ci): improve build workflow performance and security MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhances CI/CD workflows with performance optimizations and security improvements. Changes: - Add Docker build caching for faster builds (cache-from/cache-to with GitHub Actions cache) - Update Discord notification action from v1.12.0 to v1.15.5 - Pin Discord action to commit SHA (b8381b25576cb341b2af39926ab42c5056cc44ed) for security - Applied Discord action update to both build.yml and cleanup.yml workflows Performance Impact: - Build caching reduces rebuild time by reusing layers across workflow runs - Cache stored in GitHub Actions cache with mode=max for optimal storage ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/build.yml | 4 +++- .github/workflows/cleanup.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a063b4b..dbb06bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -67,9 +67,11 @@ jobs: labels: ${{ steps.meta.outputs.labels }} provenance: true sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max - name: Send notification to Discord. - uses: sarisia/actions-status-discord@v1.12.0 + uses: sarisia/actions-status-discord@b8381b25576cb341b2af39926ab42c5056cc44ed # v1.15.5 if: always() with: webhook: ${{ secrets.DISCORD_WEBHOOK }} diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index e20376d..1e53228 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -29,7 +29,7 @@ jobs: token: ${{ secrets.GHCR_TOKEN }} - name: Send notification to Discord. - uses: sarisia/actions-status-discord@v1.12.0 + uses: sarisia/actions-status-discord@b8381b25576cb341b2af39926ab42c5056cc44ed # v1.15.5 if: always() with: title: ${{ env.IMAGE_NAME }} From 07c563027ba37e96dc85b1339ffe5033f5e43fed Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 11:55:43 -0800 Subject: [PATCH 76/83] chore(ci): add Dependabot configuration for automated updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configures Dependabot to automatically monitor and update dependencies for the docker-crontab project. Configuration: - GitHub Actions: Weekly updates on Mondays at 9am PT - Docker base images: Weekly updates for alpine and docker:dind-alpine - Auto-assign to SimplicityGuy with appropriate labels - Group updates by ecosystem for cleaner PRs Monitoring: - GitHub Actions in workflows directory - Dockerfile base images (docker:dind-alpine, alpine:latest) ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/dependabot.yml | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ad450a0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,45 @@ +--- +version: 2 +updates: + # GitHub Actions dependencies + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "America/Los_Angeles" + commit-message: + prefix: "ci" + include: "scope" + labels: + - "dependencies" + - "ci" + assignees: + - "SimplicityGuy" + groups: + actions: + patterns: + - "*" + + # Docker base images + # Monitors Dockerfile for base image updates (docker:dind-alpine) + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "America/Los_Angeles" + commit-message: + prefix: "docker" + include: "scope" + labels: + - "dependencies" + - "docker" + assignees: + - "SimplicityGuy" + groups: + docker-images: + patterns: + - "*" From 5e054bb050ca56146eea94fdaf4a85ebbb95a105 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 11:55:52 -0800 Subject: [PATCH 77/83] feat(ci): add automated workflow for base image updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements automated Docker base image update workflow that checks for and creates PRs when base images are updated. Features: - Runs weekly on Mondays at 9am UTC - Manual trigger via workflow_dispatch - Checks alpine:latest and docker:dind-alpine for updates - Builds test image to verify compatibility - Creates PR with update details and digests - Auto-assigns to SimplicityGuy with appropriate labels - Discord notifications for workflow status Workflow: 1. Check for base image updates by pulling latest versions 2. Compare digests to detect changes 3. Build test image with new base images 4. Create PR with automated documentation 5. Send Discord notification Security: - Uses commit SHA pinning for create-pull-request action - Test build ensures compatibility before PR creation ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/update-dependencies.yml | 93 +++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/update-dependencies.yml diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml new file mode 100644 index 0000000..abbcbcd --- /dev/null +++ b/.github/workflows/update-dependencies.yml @@ -0,0 +1,93 @@ +--- +name: update-dependencies + +on: + schedule: + - cron: '0 9 * * 1' # Weekly on Mondays at 9am UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-base-image: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository. + uses: actions/checkout@v4 + + - name: Set up Docker Buildx. + uses: docker/setup-buildx-action@v3 + + - name: Check for base image updates. + id: check-updates + run: | + # Get current base images from Dockerfile + BUILDER_IMAGE=$(grep "^FROM alpine:latest AS builder" Dockerfile | awk '{print $2}') + RELEASE_IMAGE=$(grep "^FROM docker:.* AS release" Dockerfile | awk '{print $2}') + + echo "Current builder image: $BUILDER_IMAGE" + echo "Current release image: $RELEASE_IMAGE" + + # Pull latest versions + docker pull "$BUILDER_IMAGE" --quiet + docker pull "$RELEASE_IMAGE" --quiet + + # Get digests + BUILDER_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "$BUILDER_IMAGE" | grep -oP 'sha256:[a-f0-9]{64}' | head -1) + RELEASE_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "$RELEASE_IMAGE" | grep -oP 'sha256:[a-f0-9]{64}' | head -1) + + echo "builder_digest=$BUILDER_DIGEST" >> $GITHUB_OUTPUT + echo "release_digest=$RELEASE_DIGEST" >> $GITHUB_OUTPUT + + # Check if we need to rebuild + if [ -z "$BUILDER_DIGEST" ] || [ -z "$RELEASE_DIGEST" ]; then + echo "updates_available=true" >> $GITHUB_OUTPUT + else + echo "updates_available=false" >> $GITHUB_OUTPUT + fi + + - name: Build test image. + if: steps.check-updates.outputs.updates_available == 'true' + uses: docker/build-push-action@v6 + with: + context: . + push: false + tags: test-image:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Create Pull Request. + if: steps.check-updates.outputs.updates_available == 'true' + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'chore(deps): update base image dependencies' + title: 'chore(deps): update base image dependencies' + body: | + ## Automated Base Image Update + + This PR updates the base Docker images to their latest versions. + + ### Changes + - Builder image digest: `${{ steps.check-updates.outputs.builder_digest }}` + - Release image digest: `${{ steps.check-updates.outputs.release_digest }}` + + ### Testing + - Build test completed successfully + - Please review and test the updated images before merging + + --- + *This PR was automatically generated by the update-dependencies workflow.* + branch: chore/update-base-images + delete-branch: true + assignees: SimplicityGuy + labels: dependencies,automated,docker + + - name: Send notification to Discord. + uses: sarisia/actions-status-discord@b8381b25576cb341b2af39926ab42c5056cc44ed # v1.15.5 + if: always() + with: + webhook: ${{ secrets.DISCORD_WEBHOOK }} From efec8784dc7fb1f27864b7056a0da48c4f633dcd Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:02:25 -0800 Subject: [PATCH 78/83] chore(ci): improve workflow naming consistency - Rename "crontab" workflow to "Build" for clarity - Rename "update-dependencies" to "Update Dependencies" for consistency - Use Title Case for all workflow names to match GitHub Actions conventions --- .github/workflows/build.yml | 2 +- .github/workflows/update-dependencies.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dbb06bd..10f05b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ --- -name: crontab +name: Build on: workflow_dispatch: diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index abbcbcd..8e57f3c 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -1,5 +1,5 @@ --- -name: update-dependencies +name: Update Dependencies on: schedule: From 68fcc41ed2879a10351283a5b2dce80922267cc0 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:02:32 -0800 Subject: [PATCH 79/83] feat(ci): add automatic PR cache cleanup workflow - Automatically clean up GitHub Actions caches when PRs are closed - Prevents cache accumulation and keeps repository tidy - Uses concurrency control to prevent duplicate cleanup runs - Continues on deletion errors to ensure workflow completes - Includes 10-minute timeout for safety --- .github/workflows/cleanup-cache.yaml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/cleanup-cache.yaml diff --git a/.github/workflows/cleanup-cache.yaml b/.github/workflows/cleanup-cache.yaml new file mode 100644 index 0000000..b16b706 --- /dev/null +++ b/.github/workflows/cleanup-cache.yaml @@ -0,0 +1,36 @@ +--- +name: Cleanup Cache + +on: + pull_request: + types: + - closed + +concurrency: + group: cleanup-cache-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + cleanup: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write + steps: + - name: ๐Ÿงน Cleanup Cache + run: | + echo "Fetching list of cache keys" + cacheKeysForPR=$(gh cache list --ref "$BRANCH" --limit 100 --json id --jq ".[].id") + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh cache delete "$cacheKey" + done + echo "Done" + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge From 1008eb44b18d498d5e9b9adceab43bfcfc3b02ca Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:02:42 -0800 Subject: [PATCH 80/83] feat(ci): improve Docker image cleanup workflow Replace old cleanup workflow with improved implementation: - Migrate from snok/container-retention-policy to dataaxiom/ghcr-cleanup-action - Automatic package discovery (no hardcoded package names needed) - More flexible retention policies (keep 2 most recent tagged images) - Better handling of partial and untagged images - Remove Discord notifications to reduce dependencies - Maintain monthly cleanup schedule (15th at midnight UTC) - Add manual workflow_dispatch trigger for on-demand cleanup --- .github/workflows/cleanup-images.yml | 28 ++++++++++++++++++++ .github/workflows/cleanup.yml | 39 ---------------------------- 2 files changed, 28 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/cleanup-images.yml delete mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup-images.yml b/.github/workflows/cleanup-images.yml new file mode 100644 index 0000000..a334276 --- /dev/null +++ b/.github/workflows/cleanup-images.yml @@ -0,0 +1,28 @@ +--- +name: Cleanup Docker Images + +on: + workflow_dispatch: + schedule: + - cron: "0 0 15 * *" # Monthly on the 15th at midnight UTC + +concurrency: + group: cleanup-images + cancel-in-progress: false + +jobs: + cleanup: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + packages: write # Required to delete packages + contents: read + steps: + - name: ๐Ÿงน Cleanup Docker Images + uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 + with: + delete-partial-images: true + delete-untagged: true + keep-n-tagged: 2 + older-than: 30 days + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 1e53228..0000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: cleanup - -on: - schedule: - - cron: '0 0 15 * *' - -env: - IMAGE_NAME: ${{ github.actor }}/docker-crontab - -jobs: - cleanup-docker-crontab: - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - steps: - - name: Delete Docker images older than a month. - id: cleanup-images - uses: snok/container-retention-policy@v2 - with: - account-type: personal - cut-off: One month ago UTC - keep-at-least: 4 - skip-tags: latest - image-names: ${{ env.IMAGE_NAME }} - token: ${{ secrets.GHCR_TOKEN }} - - - name: Send notification to Discord. - uses: sarisia/actions-status-discord@b8381b25576cb341b2af39926ab42c5056cc44ed # v1.15.5 - if: always() - with: - title: ${{ env.IMAGE_NAME }} - description: | - succeded cleanup : ${{ steps.cleanup-images.outputs.deleted }} - failed cleanup : ${{ steps.cleanup-images.outputs.failed }} - webhook: ${{ secrets.DISCORD_WEBHOOK }} From 52bfb1511f63c94bcc6da2e08559a48a5bf9d500 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:08:38 -0800 Subject: [PATCH 81/83] feat(ci): add multi-platform Docker build support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add support for linux/arm64, linux/arm/v7, and linux/arm/v6 platforms - Increase job timeout from default to 90 minutes to accommodate multi-platform builds with QEMU emulation - Update both Docker Buildx setup and build steps with new platforms This enables the crontab image to run on: - linux/amd64: 64-bit x86 (Intel/AMD) - linux/arm64: 64-bit ARM (Raspberry Pi 4+, AWS Graviton) - linux/arm/v7: 32-bit ARM v7 (Raspberry Pi 2/3) - linux/arm/v6: 32-bit ARM v6 (Raspberry Pi Zero/1) ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10f05b8..a6ae9fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,7 @@ env: jobs: build-crontab: runs-on: ubuntu-latest + timeout-minutes: 90 permissions: contents: read @@ -55,13 +56,13 @@ jobs: - name: Set up Docker Buildx. uses: docker/setup-buildx-action@v3 with: - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 - name: Build and push Docker image to GitHub Container Registry. uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 6d5dd2e9ef4ed29400ab1cbb4933b1001f9f1212 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:08:39 -0800 Subject: [PATCH 82/83] fix(ci): correct package name in image cleanup workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cleanup workflow was failing with a 404 error because it was looking for package "docker-crontab" (derived from repository name) instead of the actual package name "crontab" (defined in build.yml). Add explicit package parameter to match IMAGE_NAME from build workflow: - Build workflow: IMAGE_NAME: ${{ github.actor }}/crontab - Cleanup workflow: package: crontab Fixes error: GET /users/SimplicityGuy/packages/container/docker-crontab/versions - 404 not found ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/cleanup-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cleanup-images.yml b/.github/workflows/cleanup-images.yml index a334276..3371016 100644 --- a/.github/workflows/cleanup-images.yml +++ b/.github/workflows/cleanup-images.yml @@ -21,6 +21,7 @@ jobs: - name: ๐Ÿงน Cleanup Docker Images uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 with: + package: crontab delete-partial-images: true delete-untagged: true keep-n-tagged: 2 From 06ff98c713224f76e0b33447e154c2a7c3e42472 Mon Sep 17 00:00:00 2001 From: Robert Wlodarczyk Date: Wed, 31 Dec 2025 12:15:07 -0800 Subject: [PATCH 83/83] feat(build): add multi-platform support for rq binary downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automatic platform detection to download the correct rq binary for each target architecture during Docker builds. This fixes builds for ARM platforms (arm64, armv7, armv6) which were previously failing due to hardcoded x86_64 binary. Changes: - Add TARGETPLATFORM, TARGETOS, TARGETARCH, TARGETVARIANT build args - Implement platform mapping case statement: - linux/amd64 โ†’ x86_64-unknown-linux-musl - linux/arm64 โ†’ aarch64-unknown-linux-gnu - linux/arm/v7 โ†’ armv7-unknown-linux-gnueabihf - linux/arm/v6 โ†’ arm-unknown-linux-gnueabi - Add graceful fallback to x86_64 for unknown platforms - Update hadolint ignore to include SC2086 for shell expansion This aligns the Dockerfile with the CI/CD workflow which builds for linux/amd64, linux/arm64, linux/arm/v7, and linux/arm/v6 platforms. ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- Dockerfile | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf9e4a3..eea2927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,17 +9,42 @@ LABEL org.opencontainers.image.title="crontab builder" \ org.opencontainers.image.created="$(date +'%Y-%m-%d')" \ org.opencontainers.image.base.name="docker.io/library/alpine" +# Platform arguments provided by Docker Buildx +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT + ENV RQ_VERSION=1.0.2 WORKDIR /usr/bin/rq/ -#hadolint ignore=DL3018 +#hadolint ignore=DL3018,SC2086 RUN apk update --quiet && \ apk upgrade --quiet && \ apk add --quiet --no-cache \ upx && \ rm /var/cache/apk/* && \ - wget --quiet https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ - tar -xvf rq-v${RQ_VERSION}-x86_64-unknown-linux-musl.tar.gz && \ + # Map Docker platform to rq release platform + case "${TARGETPLATFORM}" in \ + "linux/amd64") \ + RQ_PLATFORM="x86_64-unknown-linux-musl" \ + ;; \ + "linux/arm64") \ + RQ_PLATFORM="aarch64-unknown-linux-gnu" \ + ;; \ + "linux/arm/v7") \ + RQ_PLATFORM="armv7-unknown-linux-gnueabihf" \ + ;; \ + "linux/arm/v6") \ + RQ_PLATFORM="arm-unknown-linux-gnueabi" \ + ;; \ + *) \ + echo "Warning: Unknown platform ${TARGETPLATFORM}, defaulting to x86_64-unknown-linux-musl" && \ + RQ_PLATFORM="x86_64-unknown-linux-musl" \ + ;; \ + esac && \ + wget --quiet https://github.com/dflemstr/rq/releases/download/v${RQ_VERSION}/rq-v${RQ_VERSION}-${RQ_PLATFORM}.tar.gz && \ + tar -xvf rq-v${RQ_VERSION}-${RQ_PLATFORM}.tar.gz && \ upx --brute rq #hadolint ignore=DL3007