Modified daqsystemtest_integtest_bundle.sh so that it makes copies of pytest directories for failed tests

scripts/daqsystemtest_integtest_bundle.sh

@@ -102,13 +102,14 @@ if [[ "${numad_grep_output}" != "" ]]; then
 fi
 
 # other setup
-TIMESTAMP=`date '+%Y%m%d%H%M%S'`
+INITIAL_TIMESTAMP=`date '+%Y%m%d%H%M%S'`
 mkdir -p /tmp/pytest-of-${USER}
-ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/daqsystemtest_integtest_bundle_${TIMESTAMP}.log"
+ITGRUNNER_LOG_FILE="/tmp/pytest-of-${USER}/daqsystemtest_integtest_bundle_${INITIAL_TIMESTAMP}.log"
+CURRENT_PID=$$
 
 let number_of_individual_tests=0
 let test_index=0
-for TEST_NAME in ${integtest_list[@]}; do
+for TEST_NAME in "${integtest_list[@]}"; do
     if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then
         requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}`
         if [[ "${requested_test}" != "" ]]; then
@@ -124,8 +125,14 @@ let overall_test_index=0  # this is only used for user feedback
 let full_set_loop_count=0
 while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do
     let test_index=0
-    for TEST_NAME in ${integtest_list[@]}; do
+    for TEST_NAME in "${integtest_list[@]}"; do
         if [[ ${test_index} -ge ${first_test_index} && ${test_index} -le ${last_test_index} ]]; then
+            CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'`
+            # 15-Dec-2025, KAB: added the export of the following enviromental variable.  This is used
+            # by the integrationtest infrastructure to put a bread-crumb file in the directory where
+            # the test results are located.  That file, in turn, allows this script to find the directory
+            # for the current test, and make a copy of it if the test fails.
+            export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}"
             requested_test=`echo ${TEST_NAME} | egrep -i ${requested_test_names:-${TEST_NAME}}`
             if [[ "${requested_test}" != "" ]]; then
                 let individual_loop_count=0
@@ -150,8 +157,81 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do
 
                     let individual_loop_count=${individual_loop_count}+1
 
-                    if [[ ${stop_on_failure} -gt 0 ]]; then
-                        if [[ ${pytest_return_code} -ne 0 ]]; then
+                    # check if the test failed
+                    if [[ ${pytest_return_code} -ne 0 ]]; then
+                        # 15-Dec-2025, KAB: make a copy of the pytest directory. This allows
+                        # testers to take a look at the results within a reasonable time frame.
+                        # (If we can't find the "jq" JSON utility, we simply note that fact
+                        # and continue.)
+                        # This code makes use of a bread-crumb file that is created by the
+                        # integrationtest infrastructure.
+                        if [[ "`which jq 2>/dev/null`" != "" ]]; then
+                            current_pytest_rundir=""
+                            mapfile -t bundle_info_files < <(find "/tmp/pytest-of-${USER}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}')
+                            for info_file in "${bundle_info_files[@]}"; do
+                                script_start_time=`jq -r .bundle_script_start_time ${info_file}`
+                                script_pid=`jq -r .bundle_script_process_id ${info_file}`
+                                individual_test_start_time=`jq -r .individual_test_start_time ${info_file}`
+                                if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \
+                                       [[ ${script_pid} -eq ${CURRENT_PID} ]] && \
+                                       [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then
+                                    current_pytest_rundir=$info_file
+                                    break
+                                fi
+                            done
+
+                            was_successfully_copied=""
+                            if [[ "${current_pytest_rundir}" != "" ]]; then
+                                pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname`
+                                if [[ "${pytest_tmpdir}" != "" ]]; then
+                                    pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname`
+                                    pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename`
+                                    if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then
+                                        new_dir="${pytest_rootdir}/failed-${pytest_basedir}"
+                                        echo ""
+                                        echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535"
+                                        cp -pR "${pytest_tmpdir}" "${new_dir}"
+                                        if [[ $? == 0 ]]; then
+                                            was_successfully_copied="yes"
+                                            # 18-Dec-2025, KAB: added the removal of the "current" symbolic links
+                                            # from inside the copied directory (since they get broken in the copying)
+                                            rm -f "${new_dir}/configcurrent"
+                                            rm -f "${new_dir}/runcurrent"
+                                        fi
+                                    fi
+                                fi
+                            fi
+                            if [[ "${was_successfully_copied}" == "" ]]; then
+                                echo ""
+                                echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1"
+                            fi
+                        else
+                            echo ""
+                            echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1"
+                        fi
+
+                        # remove stale and surplus directories from failed tests
+                        test_dirs_to_remove=()
+                        mapfile -t all_failed_test_dirs < <(find /tmp/pytest-of-${USER} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-')
+                        surplus_dirs=("${all_failed_test_dirs[@]:10}")
+                        for test_dir in "${surplus_dirs[@]}"; do
+                            test_dirs_to_remove+=(${test_dir})
+                        done
+                        stale_failed_test_dirs=(`find /tmp/pytest-of-${USER} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`)
+                        for test_dir in "${stale_failed_test_dirs[@]}"; do
+                            test_dirs_to_remove+=(${test_dir})
+                        done
+                        if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then
+                            echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535"
+                            for test_dir in "${test_dirs_to_remove[@]}"; do
+                                if [[ -e "${test_dir}" ]]; then
+                                    rm -rf "${test_dir}"
+                                fi
+                            done
+                        fi
+
+                        # exit out of this script if the user has requested that we stop on a failure
+                        if [[ ${stop_on_failure} -gt 0 ]]; then
                             break 3
                         fi
                     fi