From cedd53abf900db8124fae485b3199df71b99b606 Mon Sep 17 00:00:00 2001 From: Christopher Slycord Date: Wed, 12 Nov 2025 21:49:53 +0900 Subject: [PATCH 1/5] Ensure that we don't skip any commands, in case the total is not divisible by the chosen maximum --- parallel-bash.bash | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/parallel-bash.bash b/parallel-bash.bash index c62b9ab..493c0af 100755 --- a/parallel-bash.bash +++ b/parallel-bash.bash @@ -93,6 +93,15 @@ _process_arguments::parallel-bash() { ;; esac + # when total commands is not divisible by $NO_OF_JOBS + # some remain unprocessed in the end + if [ -n "${cmds}" ]; then + job=0 + # all hail the eval lord + eval "${cmds}" & + cmds="" + fi + # this is probably pointless as the processes might be already completed before even reaching this point # todo: fix this declare status @@ -143,4 +152,4 @@ parallel-bash() { return 0 } -parallel-bash "${@}" || exit 1 +parallel-bash "${@}" || exit 1 \ No newline at end of file From b4b94247b048a3eba1c8d5d46a7d56d33d89f825 Mon Sep 17 00:00:00 2001 From: Christopher Slycord Date: Thu, 13 Nov 2025 00:55:23 +0900 Subject: [PATCH 2/5] added back ending newline --- parallel-bash.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallel-bash.bash b/parallel-bash.bash index 493c0af..fc3ff10 100755 --- a/parallel-bash.bash +++ b/parallel-bash.bash @@ -152,4 +152,4 @@ parallel-bash() { return 0 } -parallel-bash "${@}" || exit 1 \ No newline at end of file +parallel-bash "${@}" || exit 1 From 85191883164ea9df4ec8a6d68338aa8d8dcb1db6 Mon Sep 17 00:00:00 2001 From: Christopher Slycord Date: Thu, 13 Nov 2025 11:00:47 +0900 Subject: [PATCH 3/5] Refactor parallel-bash script for clarity and structure Refactor the job execution into its own function. Removes code duplication, since now the code is called in two sections of the script --- parallel-bash.bash | 229 ++++++++++++++++++++++----------------------- 1 file changed, 114 insertions(+), 115 deletions(-) diff --git a/parallel-bash.bash b/parallel-bash.bash index fc3ff10..ee4fccc 100755 --- a/parallel-bash.bash +++ b/parallel-bash.bash @@ -3,12 +3,12 @@ # Also supports functions _usage::parallel-bash() { - printf "%b\n" \ - "Parallel processing commands in pure bash ( like xargs ) + printf "%b\n" \ + "Parallel processing commands in pure bash ( like xargs ) Usage: something | ${0##*/} -p 5 echo - ${0##*/} -p 5 echo < some_file - ${0##*/} -p 5 echo <<< 'some string' + ${0##*/} -p 5 echo < some_file + ${0##*/} -p 5 echo <<< 'some string' The keyword '{}' can be used to for command inputs. @@ -18,138 +18,137 @@ e.g: something | ${0##*/} -p 5 echo {} Optional flags: - -k | -kc | --kill-children-processes => Kill children processes created when command is manually interrupted. + -k | -kc | --kill-children-processes => Kill children processes created when command is manually interrupted. - -p | --parallel-jobs => Number of parallel processes. Default value is 1. + -p | --parallel-jobs => Number of parallel processes. Default value is 1. - -D | --debug => Show debug trace. + -D | --debug => Show debug trace. - -h | --help => Show this help." - exit 0 + -h | --help => Show this help." + exit 0 } _setup_arguments::parallel-bash() { - while [[ $# -gt 0 ]]; do - case "${1}" in - -h | --help) _usage::parallel-bash ;; - -D | --debug) set -x ;; - -p | --parallel-jobs) - if [[ ${2} -gt 0 ]]; then - NO_OF_JOBS="${2}" && shift - else - printf "\nError: -P only takes as positive integer as argument.\n" - return 1 - fi - ;; - -k | -kc | --kill-children-processes) - KILL_CHILD_PROCESSES=true - ;; - *) - CMD_ARRAY="" - # all the args given after -c is taken as command input - for cmd in "${@}"; do CMD_ARRAY+="\"${cmd}\" "; done - break - ;; - esac - shift - done - - return 0 + while [[ $# -gt 0 ]]; do + case "${1}" in + -h | --help) _usage::parallel-bash ;; + -D | --debug) set -x ;; + -p | --parallel-jobs) + if [[ ${2} -gt 0 ]]; then + NO_OF_JOBS="${2}" && shift + else + printf "\nError: -P only takes as positive integer as argument.\n" + return 1 + fi + ;; + -k | -kc | --kill-children-processes) + KILL_CHILD_PROCESSES=true + ;; + *) + CMD_ARRAY="" + # all the args given after -c is taken as command input + for cmd in "${@}"; do CMD_ARRAY+="\"${cmd}\" "; done + break + ;; + esac + shift + done + + return 0 } _process_arguments::parallel-bash() { - declare job=0 cmds="" - - # a wrapper function - # takes 1 argument - _execute::_process_arguments::parallel-bash() { - # job, no_of_jobs_final ans cmds is from parent function - ((job += 1)) - # not using arrays because it is slow - # `;` is added to last to prevent stopping the execution because of a failed process - export "cmds+=${1:-:} ; " - # when job == no_of_jobs_final, then reset it and then again start appending from job 1 - [[ ${job} -eq "${NO_OF_JOBS}" ]] && { - job=0 - # all hail the eval lord - eval "${cmds}" & - cmds="" - } - } - - # iterate over both input arrays - # then pass formed string for _execute::_process_arguments::parallel-bash - case "${CMD_ARRAY}" in - *'{}'*) - while IFS= read -r line; do - # If CMD_ARRAY array contains {}, then replace it with the input - _execute::_process_arguments::parallel-bash "${CMD_ARRAY//\{\}/\"${line}\"}" - done - ;; - *) - while IFS= read -r line; do - _execute::_process_arguments::parallel-bash "${CMD_ARRAY} \"${line}\"" - done - ;; - esac - - # when total commands is not divisible by $NO_OF_JOBS - # some remain unprocessed in the end - if [ -n "${cmds}" ]; then - job=0 - # all hail the eval lord - eval "${cmds}" & - cmds="" - fi - - # this is probably pointless as the processes might be already completed before even reaching this point - # todo: fix this - declare status - wait || status=1 - - return "${status:-0}" + declare job=0 cmds="" + + # function for running stored processes + # runs tasks from cmds variable and resets cmds plus job count to default + _execute::_run_processes::parallel-bash() { + job=0 + # all hail the eval lord + eval "${cmds}" & + cmds="" + } + + # a wrapper function + # takes 1 argument + _execute::_process_arguments::parallel-bash() { + # job, no_of_jobs_final ans cmds is from parent function + ((job += 1)) + # not using arrays because it is slow + # `;` is added to last to prevent stopping the execution because of a failed process + export "cmds+=${1:-:} ; " + # when job == no_of_jobs_final, then reset it and then again start appending from job 1 + [[ ${job} -eq "${NO_OF_JOBS}" ]] && _execute::_run_processes::parallel-bash + } + + # iterate over both input arrays + # then pass formed string for _execute::_process_arguments::parallel-bash + case "${CMD_ARRAY}" in + *'{}'*) + while IFS= read -r line; do + # If CMD_ARRAY array contains {}, then replace it with the input + _execute::_process_arguments::parallel-bash "${CMD_ARRAY//\{\}/\"${line}\"}" + done + ;; + *) + while IFS= read -r line; do + _execute::_process_arguments::parallel-bash "${CMD_ARRAY} \"${line}\"" + done + ;; + esac + + # when total commands is not divisible by $NO_OF_JOBS + # some remain unprocessed in the end + [ -n "${cmds}" ] && _execute::_run_processes::parallel-bash + + # this is probably pointless as the processes might be already completed before even reaching this point + # todo: fix this + declare status + wait || status=1 + + return "${status:-0}" } _cleanup::parallel-bash() { - { - # print messages if exited manually - export abnormal_exit && if [[ -n ${abnormal_exit} ]]; then - p_print() { printf "%b\n" "${1}"; } - else - p_print() { :; } - fi - - p_printf "\n\nparallel-bash exited manually." - if [[ ${KILL_CHILD_PROCESSES} = "true" ]]; then - p_printf "Killing child processes." - # this kills the script including all the child processes - kill -- -$$ & - else - p_printf "Not killing child processes." - fi - } 2>| /dev/null || : - return 0 + { + # print messages if exited manually + export abnormal_exit && if [[ -n ${abnormal_exit} ]]; then + p_print() { printf "%b\n" "${1}"; } + else + p_print() { :; } + fi + + p_printf "\n\nparallel-bash exited manually." + if [[ ${KILL_CHILD_PROCESSES} = "true" ]]; then + p_printf "Killing child processes." + # this kills the script including all the child processes + kill -- -$$ & + else + p_printf "Not killing child processes." + fi + } 2>| /dev/null || : + return 0 } parallel-bash() { - [[ $# = 0 ]] && _usage::parallel-bash + [[ $# = 0 ]] && _usage::parallel-bash - ! [[ ${BASH_VERSINFO:-0} -ge 3 ]] && - printf "Bash version lower than 3.x not supported.\n" && return 1 + ! [[ ${BASH_VERSINFO:-0} -ge 3 ]] && + printf "Bash version lower than 3.x not supported.\n" && return 1 - set -o errexit -o noclobber -o pipefail + set -o errexit -o noclobber -o pipefail - trap 'abnormal_exit="1"; exit' INT TERM - trap '_cleanup::parallel-bash' EXIT - trap '' TSTP # ignore ctrl + z + trap 'abnormal_exit="1"; exit' INT TERM + trap '_cleanup::parallel-bash' EXIT + trap '' TSTP # ignore ctrl + z - declare NO_OF_JOBS=1 CMD_ARRAY MAIN_PID KILL_CHILD_PROCESSES - _setup_arguments::parallel-bash "${@}" || return 1 + declare NO_OF_JOBS=1 CMD_ARRAY MAIN_PID KILL_CHILD_PROCESSES + _setup_arguments::parallel-bash "${@}" || return 1 - export MAIN_PID="$$" + export MAIN_PID="$$" - _process_arguments::parallel-bash || return 1 - return 0 + _process_arguments::parallel-bash || return 1 + return 0 } parallel-bash "${@}" || exit 1 From af7a348d3e10fdeb8debf7e284497ce125a3c543 Mon Sep 17 00:00:00 2001 From: Christopher Slycord Date: Thu, 13 Nov 2025 11:07:24 +0900 Subject: [PATCH 4/5] Revert "Refactor parallel-bash script for clarity and structure" This reverts commit 85191883164ea9df4ec8a6d68338aa8d8dcb1db6. --- parallel-bash.bash | 229 +++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 114 deletions(-) diff --git a/parallel-bash.bash b/parallel-bash.bash index ee4fccc..fc3ff10 100755 --- a/parallel-bash.bash +++ b/parallel-bash.bash @@ -3,12 +3,12 @@ # Also supports functions _usage::parallel-bash() { - printf "%b\n" \ - "Parallel processing commands in pure bash ( like xargs ) + printf "%b\n" \ + "Parallel processing commands in pure bash ( like xargs ) Usage: something | ${0##*/} -p 5 echo - ${0##*/} -p 5 echo < some_file - ${0##*/} -p 5 echo <<< 'some string' + ${0##*/} -p 5 echo < some_file + ${0##*/} -p 5 echo <<< 'some string' The keyword '{}' can be used to for command inputs. @@ -18,137 +18,138 @@ e.g: something | ${0##*/} -p 5 echo {} Optional flags: - -k | -kc | --kill-children-processes => Kill children processes created when command is manually interrupted. + -k | -kc | --kill-children-processes => Kill children processes created when command is manually interrupted. - -p | --parallel-jobs => Number of parallel processes. Default value is 1. + -p | --parallel-jobs => Number of parallel processes. Default value is 1. - -D | --debug => Show debug trace. + -D | --debug => Show debug trace. - -h | --help => Show this help." - exit 0 + -h | --help => Show this help." + exit 0 } _setup_arguments::parallel-bash() { - while [[ $# -gt 0 ]]; do - case "${1}" in - -h | --help) _usage::parallel-bash ;; - -D | --debug) set -x ;; - -p | --parallel-jobs) - if [[ ${2} -gt 0 ]]; then - NO_OF_JOBS="${2}" && shift - else - printf "\nError: -P only takes as positive integer as argument.\n" - return 1 - fi - ;; - -k | -kc | --kill-children-processes) - KILL_CHILD_PROCESSES=true - ;; - *) - CMD_ARRAY="" - # all the args given after -c is taken as command input - for cmd in "${@}"; do CMD_ARRAY+="\"${cmd}\" "; done - break - ;; - esac - shift - done - - return 0 + while [[ $# -gt 0 ]]; do + case "${1}" in + -h | --help) _usage::parallel-bash ;; + -D | --debug) set -x ;; + -p | --parallel-jobs) + if [[ ${2} -gt 0 ]]; then + NO_OF_JOBS="${2}" && shift + else + printf "\nError: -P only takes as positive integer as argument.\n" + return 1 + fi + ;; + -k | -kc | --kill-children-processes) + KILL_CHILD_PROCESSES=true + ;; + *) + CMD_ARRAY="" + # all the args given after -c is taken as command input + for cmd in "${@}"; do CMD_ARRAY+="\"${cmd}\" "; done + break + ;; + esac + shift + done + + return 0 } _process_arguments::parallel-bash() { - declare job=0 cmds="" - - # function for running stored processes - # runs tasks from cmds variable and resets cmds plus job count to default - _execute::_run_processes::parallel-bash() { - job=0 - # all hail the eval lord - eval "${cmds}" & - cmds="" - } - - # a wrapper function - # takes 1 argument - _execute::_process_arguments::parallel-bash() { - # job, no_of_jobs_final ans cmds is from parent function - ((job += 1)) - # not using arrays because it is slow - # `;` is added to last to prevent stopping the execution because of a failed process - export "cmds+=${1:-:} ; " - # when job == no_of_jobs_final, then reset it and then again start appending from job 1 - [[ ${job} -eq "${NO_OF_JOBS}" ]] && _execute::_run_processes::parallel-bash - } - - # iterate over both input arrays - # then pass formed string for _execute::_process_arguments::parallel-bash - case "${CMD_ARRAY}" in - *'{}'*) - while IFS= read -r line; do - # If CMD_ARRAY array contains {}, then replace it with the input - _execute::_process_arguments::parallel-bash "${CMD_ARRAY//\{\}/\"${line}\"}" - done - ;; - *) - while IFS= read -r line; do - _execute::_process_arguments::parallel-bash "${CMD_ARRAY} \"${line}\"" - done - ;; - esac - - # when total commands is not divisible by $NO_OF_JOBS - # some remain unprocessed in the end - [ -n "${cmds}" ] && _execute::_run_processes::parallel-bash - - # this is probably pointless as the processes might be already completed before even reaching this point - # todo: fix this - declare status - wait || status=1 - - return "${status:-0}" + declare job=0 cmds="" + + # a wrapper function + # takes 1 argument + _execute::_process_arguments::parallel-bash() { + # job, no_of_jobs_final ans cmds is from parent function + ((job += 1)) + # not using arrays because it is slow + # `;` is added to last to prevent stopping the execution because of a failed process + export "cmds+=${1:-:} ; " + # when job == no_of_jobs_final, then reset it and then again start appending from job 1 + [[ ${job} -eq "${NO_OF_JOBS}" ]] && { + job=0 + # all hail the eval lord + eval "${cmds}" & + cmds="" + } + } + + # iterate over both input arrays + # then pass formed string for _execute::_process_arguments::parallel-bash + case "${CMD_ARRAY}" in + *'{}'*) + while IFS= read -r line; do + # If CMD_ARRAY array contains {}, then replace it with the input + _execute::_process_arguments::parallel-bash "${CMD_ARRAY//\{\}/\"${line}\"}" + done + ;; + *) + while IFS= read -r line; do + _execute::_process_arguments::parallel-bash "${CMD_ARRAY} \"${line}\"" + done + ;; + esac + + # when total commands is not divisible by $NO_OF_JOBS + # some remain unprocessed in the end + if [ -n "${cmds}" ]; then + job=0 + # all hail the eval lord + eval "${cmds}" & + cmds="" + fi + + # this is probably pointless as the processes might be already completed before even reaching this point + # todo: fix this + declare status + wait || status=1 + + return "${status:-0}" } _cleanup::parallel-bash() { - { - # print messages if exited manually - export abnormal_exit && if [[ -n ${abnormal_exit} ]]; then - p_print() { printf "%b\n" "${1}"; } - else - p_print() { :; } - fi - - p_printf "\n\nparallel-bash exited manually." - if [[ ${KILL_CHILD_PROCESSES} = "true" ]]; then - p_printf "Killing child processes." - # this kills the script including all the child processes - kill -- -$$ & - else - p_printf "Not killing child processes." - fi - } 2>| /dev/null || : - return 0 + { + # print messages if exited manually + export abnormal_exit && if [[ -n ${abnormal_exit} ]]; then + p_print() { printf "%b\n" "${1}"; } + else + p_print() { :; } + fi + + p_printf "\n\nparallel-bash exited manually." + if [[ ${KILL_CHILD_PROCESSES} = "true" ]]; then + p_printf "Killing child processes." + # this kills the script including all the child processes + kill -- -$$ & + else + p_printf "Not killing child processes." + fi + } 2>| /dev/null || : + return 0 } parallel-bash() { - [[ $# = 0 ]] && _usage::parallel-bash + [[ $# = 0 ]] && _usage::parallel-bash - ! [[ ${BASH_VERSINFO:-0} -ge 3 ]] && - printf "Bash version lower than 3.x not supported.\n" && return 1 + ! [[ ${BASH_VERSINFO:-0} -ge 3 ]] && + printf "Bash version lower than 3.x not supported.\n" && return 1 - set -o errexit -o noclobber -o pipefail + set -o errexit -o noclobber -o pipefail - trap 'abnormal_exit="1"; exit' INT TERM - trap '_cleanup::parallel-bash' EXIT - trap '' TSTP # ignore ctrl + z + trap 'abnormal_exit="1"; exit' INT TERM + trap '_cleanup::parallel-bash' EXIT + trap '' TSTP # ignore ctrl + z - declare NO_OF_JOBS=1 CMD_ARRAY MAIN_PID KILL_CHILD_PROCESSES - _setup_arguments::parallel-bash "${@}" || return 1 + declare NO_OF_JOBS=1 CMD_ARRAY MAIN_PID KILL_CHILD_PROCESSES + _setup_arguments::parallel-bash "${@}" || return 1 - export MAIN_PID="$$" + export MAIN_PID="$$" - _process_arguments::parallel-bash || return 1 - return 0 + _process_arguments::parallel-bash || return 1 + return 0 } parallel-bash "${@}" || exit 1 From f89ca33d0f3d6bbc7432cd7a2efcc33793608fa1 Mon Sep 17 00:00:00 2001 From: Christopher Slycord Date: Thu, 13 Nov 2025 11:18:11 +0900 Subject: [PATCH 5/5] parallel-bash.bash: Move process exec to function Refactor parallel-bash.bash to place process/task execution into a function Because the code is now called twice, this removes code duplication. --- parallel-bash.bash | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/parallel-bash.bash b/parallel-bash.bash index fc3ff10..e1d7ca7 100755 --- a/parallel-bash.bash +++ b/parallel-bash.bash @@ -60,6 +60,15 @@ _setup_arguments::parallel-bash() { _process_arguments::parallel-bash() { declare job=0 cmds="" + # function for running stored processes + # runs tasks from cmds variable and resets cmds plus job count to default + _execute::_run_processes::parallel-bash() { + job=0 + # all hail the eval lord + eval "${cmds}" & + cmds="" + } + # a wrapper function # takes 1 argument _execute::_process_arguments::parallel-bash() { @@ -69,12 +78,7 @@ _process_arguments::parallel-bash() { # `;` is added to last to prevent stopping the execution because of a failed process export "cmds+=${1:-:} ; " # when job == no_of_jobs_final, then reset it and then again start appending from job 1 - [[ ${job} -eq "${NO_OF_JOBS}" ]] && { - job=0 - # all hail the eval lord - eval "${cmds}" & - cmds="" - } + [[ ${job} -eq "${NO_OF_JOBS}" ]] && _execute::_run_processes::parallel-bash } # iterate over both input arrays @@ -95,12 +99,7 @@ _process_arguments::parallel-bash() { # when total commands is not divisible by $NO_OF_JOBS # some remain unprocessed in the end - if [ -n "${cmds}" ]; then - job=0 - # all hail the eval lord - eval "${cmds}" & - cmds="" - fi + [ -n "${cmds}" ] && _execute::_run_processes::parallel-bash # this is probably pointless as the processes might be already completed before even reaching this point # todo: fix this