From 8018cd648da4b07d3549094a4c03ffd8379a69a2 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Mon, 15 Dec 2025 17:05:02 -0600 Subject: [PATCH 1/4] Modified daqsystemtest_integtest_bundle.sh so that it makes copies of the logfile directories for failed tests. --- scripts/daqsystemtest_integtest_bundle.sh | 84 +++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/scripts/daqsystemtest_integtest_bundle.sh b/scripts/daqsystemtest_integtest_bundle.sh index ed17718..d018709 100755 --- a/scripts/daqsystemtest_integtest_bundle.sh +++ b/scripts/daqsystemtest_integtest_bundle.sh @@ -102,9 +102,10 @@ if [[ "${numad_grep_output}" != "" ]]; then fi # other setup -TIMESTAMP=`date '+%Y%m%d%H%M%S'` +INITIAL_TIMESTAMP=`date '+%Y%m%d%H%M%S'` mkdir -p /tmp/pytest-of-${USER} -ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/daqsystemtest_integtest_bundle_${TIMESTAMP}.log" +ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/daqsystemtest_integtest_bundle_${INITIAL_TIMESTAMP}.log" +CURRENT_PID=$$ let number_of_individual_tests=0 let test_index=0 @@ -126,6 +127,12 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 for TEST_NAME in ${integtest_list[@]}; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then + CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'` + # 15-Dec-2025, KAB: added the export of the following enviromental variable. This is used + # by the integrationtest infrastructure to put a bread-crumb file in the directory where + # the test results are located. That file, in turn, allows this script to find the directory + # for the current test, and make a copy of it if the test fails. + export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}" requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` if [[ "${requested_test}" != "" ]]; then let individual_loop_count=0 @@ -150,11 +157,80 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let individual_loop_count=${individual_loop_count}+1 - if [[ ${stop_on_failure} -gt 0 ]]; then - if [[ ${pytest_return_code} -ne 0 ]]; then + # check if the test failed + if [[ ${pytest_return_code} -ne 0 ]]; then + # 15-Dec-2025, KAB: make a copy of the pytest directory. This allows + # testers to take a look at the results within a reasonable time frame. + # (If we can't find the "jq" JSON utility, we simply note that fact + # and continue.) + # This code makes use of a bread-crumb file that is created by the + # integrationtest infrastructure. + if [[ "`which jq 2>/dev/null`" != "" ]]; then + current_pytest_rundir="" + bundle_info_files=(`find /tmp/pytest-of-${USER} -type f -print | grep bundle_script_info.json | xargs -r ls -1t`) + for info_file in ${bundle_info_files[@]}; do + script_start_time=`jq -r .bundle_script_start_time ${info_file}` + script_pid=`jq -r .bundle_script_process_id ${info_file}` + individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` + if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \ + [[ ${script_pid} -eq ${CURRENT_PID} ]] && \ + [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then + current_pytest_rundir=$info_file + break + fi + done + + was_successfully_copied="" + if [[ "${current_pytest_rundir}" != "" ]]; then + pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname` + if [[ "${pytest_tmpdir}" != "" ]]; then + pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname` + pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename` + if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then + new_dir="${pytest_rootdir}/failed-${pytest_basedir}" + echo "" + echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" + cp -pR ${pytest_tmpdir} ${new_dir} + if [[ $? == 0 ]]; then + was_successfully_copied="yes" + fi + fi + fi + fi + if [[ "${was_successfully_copied}" == "" ]]; then + echo "" + echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1" + fi + else + echo "" + echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" + fi + + # exit out of this script if the user has requested that we stop on a failure + if [[ ${stop_on_failure} -gt 0 ]]; then break 3 fi fi + + # remove stale and surplus directories from failed tests + test_dirs_to_remove=() + all_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -print | grep 'failed-' | xargs -r ls -1dt`) + surplus_dirs=("${all_failed_test_dirs[@]:10}") + for test_dir in ${surplus_dirs[@]}; do + test_dirs_to_remove+=(${test_dir}) + done + stale_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) + for test_dir in ${stale_failed_test_dirs[@]}; do + test_dirs_to_remove+=(${test_dir}) + done + if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" + for test_dir in ${test_dirs_to_remove[@]}; do + if [[ -e ${test_dir} ]]; then + rm -rf ${test_dir} + fi + done + fi done fi fi From 2322a89fb5c4efb00b179a520e1eb181c06bb61f Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Fri, 19 Dec 2025 08:18:13 -0600 Subject: [PATCH 2/4] Added removal of symlinks when we copy a failed pytest dir. --- scripts/daqsystemtest_integtest_bundle.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/daqsystemtest_integtest_bundle.sh b/scripts/daqsystemtest_integtest_bundle.sh index d018709..9f4c399 100755 --- a/scripts/daqsystemtest_integtest_bundle.sh +++ b/scripts/daqsystemtest_integtest_bundle.sh @@ -193,6 +193,10 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do cp -pR ${pytest_tmpdir} ${new_dir} if [[ $? == 0 ]]; then was_successfully_copied="yes" + # 18-Dec-2025, KAB: added the removal of the "current" symbolic links + # from inside the copied directory (since they get broken in the copying) + rm -f ${new_dir}/configcurrent + rm -f ${new_dir}/runcurrent fi fi fi From 3d5f3a0b55d5859560f7b81d75520ac94e32a0d0 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Mon, 22 Dec 2025 11:28:58 -0600 Subject: [PATCH 3/4] added quotes around various shell env var expansions in daqsystemtest_integtest_bundle.sh --- scripts/daqsystemtest_integtest_bundle.sh | 52 +++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/scripts/daqsystemtest_integtest_bundle.sh b/scripts/daqsystemtest_integtest_bundle.sh index 9f4c399..2e5d0ad 100755 --- a/scripts/daqsystemtest_integtest_bundle.sh +++ b/scripts/daqsystemtest_integtest_bundle.sh @@ -109,7 +109,7 @@ CURRENT_PID=$$ let number_of_individual_tests=0 let test_index=0 -for TEST_NAME in ${integtest_list[@]}; do +for TEST_NAME in "${integtest_list[@]}"; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}` if [[ "${requested_test}" != "" ]]; then @@ -125,7 +125,7 @@ let overall_test_index=0 # this is only used for user feedback let full_set_loop_count=0 while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 - for TEST_NAME in ${integtest_list[@]}; do + for TEST_NAME in "${integtest_list[@]}"; do if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'` # 15-Dec-2025, KAB: added the export of the following enviromental variable. This is used @@ -168,7 +168,7 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do if [[ "`which jq 2>/dev/null`" != "" ]]; then current_pytest_rundir="" bundle_info_files=(`find /tmp/pytest-of-${USER} -type f -print | grep bundle_script_info.json | xargs -r ls -1t`) - for info_file in ${bundle_info_files[@]}; do + for info_file in "${bundle_info_files[@]}"; do script_start_time=`jq -r .bundle_script_start_time ${info_file}` script_pid=`jq -r .bundle_script_process_id ${info_file}` individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` @@ -190,13 +190,13 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do new_dir="${pytest_rootdir}/failed-${pytest_basedir}" echo "" echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" - cp -pR ${pytest_tmpdir} ${new_dir} + cp -pR "${pytest_tmpdir}" "${new_dir}" if [[ $? == 0 ]]; then was_successfully_copied="yes" # 18-Dec-2025, KAB: added the removal of the "current" symbolic links # from inside the copied directory (since they get broken in the copying) - rm -f ${new_dir}/configcurrent - rm -f ${new_dir}/runcurrent + rm -f "${new_dir}/configcurrent" + rm -f "${new_dir}/runcurrent" fi fi fi @@ -210,31 +210,31 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" fi + # remove stale and surplus directories from failed tests + test_dirs_to_remove=() + all_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -print | grep 'failed-' | xargs -r ls -1dt`) + surplus_dirs=("${all_failed_test_dirs[@]:10}") + for test_dir in "${surplus_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + stale_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) + for test_dir in "${stale_failed_test_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" + for test_dir in "${test_dirs_to_remove[@]}"; do + if [[ -e "${test_dir}" ]]; then + rm -rf "${test_dir}" + fi + done + fi + # exit out of this script if the user has requested that we stop on a failure if [[ ${stop_on_failure} -gt 0 ]]; then break 3 fi fi - - # remove stale and surplus directories from failed tests - test_dirs_to_remove=() - all_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -print | grep 'failed-' | xargs -r ls -1dt`) - surplus_dirs=("${all_failed_test_dirs[@]:10}") - for test_dir in ${surplus_dirs[@]}; do - test_dirs_to_remove+=(${test_dir}) - done - stale_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) - for test_dir in ${stale_failed_test_dirs[@]}; do - test_dirs_to_remove+=(${test_dir}) - done - if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then - echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" - for test_dir in ${test_dirs_to_remove[@]}; do - if [[ -e ${test_dir} ]]; then - rm -rf ${test_dir} - fi - done - fi done fi fi From 4b7c24ac3e8e03b19440e2a1861a6a5ce00286a2 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Mon, 29 Dec 2025 14:13:22 -0600 Subject: [PATCH 4/4] Updated 'find' calls in daqsystemtest_integtest_bundle.sh to make use of the 'mapfile' command, as suggested by Michal. --- scripts/daqsystemtest_integtest_bundle.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/daqsystemtest_integtest_bundle.sh b/scripts/daqsystemtest_integtest_bundle.sh index 2e5d0ad..c73f3bd 100755 --- a/scripts/daqsystemtest_integtest_bundle.sh +++ b/scripts/daqsystemtest_integtest_bundle.sh @@ -167,7 +167,7 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do # integrationtest infrastructure. if [[ "`which jq 2>/dev/null`" != "" ]]; then current_pytest_rundir="" - bundle_info_files=(`find /tmp/pytest-of-${USER} -type f -print | grep bundle_script_info.json | xargs -r ls -1t`) + mapfile -t bundle_info_files < <(find "/tmp/pytest-of-${USER}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}') for info_file in "${bundle_info_files[@]}"; do script_start_time=`jq -r .bundle_script_start_time ${info_file}` script_pid=`jq -r .bundle_script_process_id ${info_file}` @@ -212,7 +212,7 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do # remove stale and surplus directories from failed tests test_dirs_to_remove=() - all_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -print | grep 'failed-' | xargs -r ls -1dt`) + mapfile -t all_failed_test_dirs < <(find /tmp/pytest-of-${USER} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-') surplus_dirs=("${all_failed_test_dirs[@]:10}") for test_dir in "${surplus_dirs[@]}"; do test_dirs_to_remove+=(${test_dir})