Provide the ability to delete (remove) HDF5 files at the end of integration tests

src/integrationtest/data_classes.py

@@ -107,9 +107,9 @@ class integtest_params_for_predefined_dunedaq_config(integtest_param_base_class)
 
 @dataclass
 class CreateConfigResult:
-    config: integtest_param_base_class
-    config_dir: str
-    config_file: str
+    integtest_params: integtest_param_base_class
+    dunedaq_config_dir: str
+    dunedaq_config_file: str
     log_file: str
     data_dirs: list[str]
     tpstream_data_dirs: list[str]

src/integrationtest/integrationtest_commandline.py

@@ -46,7 +46,7 @@ def pytest_addoption(parser):
         "--integtest-verbosity",
         action="store",
         default=3,
-        help="The volume of messages that are printed out by the integration test infrastructure",
+        help="This controls the volume of messages that are printed out by the integration test infrastructure (1 is lowest, 6 is highest)",
         required=False
     )
     parser.addoption(
@@ -56,6 +56,13 @@ def pytest_addoption(parser):
         help="A phrase that, if found in run control messages, will trigger the printout of all RC messages",
         required=False
     )
+    parser.addoption(
+        "--remove-hdf5-files",
+        action="store",
+        default=None,
+        help="Whether to remove HDF5 (data) files when the testing has finished (this over-rides the choice in the integtest). 'True' forces files to be removed, 'False' forces files to be kept.",
+        required=False
+    )
 
 def pytest_configure(config):
     for opt in ("--dunerc-path",):

src/integrationtest/integrationtest_drunc.py

@@ -410,9 +410,9 @@ def apply_update(obj, substitution):
             pass
 
     result = CreateConfigResult(
-        config=integtest_params,
-        config_dir=config_dir,
-        config_file=config_db,
+        integtest_params=integtest_params,
+        dunedaq_config_dir=config_dir,
+        dunedaq_config_file=config_db,
         log_file=logfile,
         data_dirs=rawdata_dirs,
         tpstream_data_dirs=tpstream_dirs,
@@ -442,8 +442,8 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
     integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity"))
 
     if no_integtest_connsvc and \
-       isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config):
-        create_config_files.config.connsvc_control = ConnSvcControl.NONE
+       isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config):
+        create_config_files.integtest_params.connsvc_control = ConnSvcControl.NONE
 
     run_dir = tmp_path_factory.mktemp("run")
 
@@ -497,36 +497,36 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
     # start the Connectivity Service, if requested (only supported for generated dune-daq configs, for now)
     connsvc_obj = None
     if (
-        isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config)
-        and create_config_files.config.connsvc_control == ConnSvcControl.INTEGRATIONTEST
+        isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config)
+        and create_config_files.integtest_params.connsvc_control == ConnSvcControl.INTEGRATIONTEST
     ):
         # start connsvc
         if integtest_verbosity_level >= IntegtestVerbosityLevels.full_output:
             print(
-                f"Starting Connectivity Service on port {create_config_files.config.connsvc_port}"
+                f"Starting Connectivity Service on port {create_config_files.integtest_params.connsvc_port}"
             )
 
         connsvc_env = os.environ.copy()
-        if create_config_files.config.connsvc_debug_level is not None:
+        if create_config_files.integtest_params.connsvc_debug_level is not None:
             connsvc_env["CONNECTION_FLASK_DEBUG"] = str(
-                create_config_files.config.connsvc_debug_level
+                create_config_files.integtest_params.connsvc_debug_level
             )
 
         connsvc_log = open(
             run_dir
-            / f"log_{getpass.getuser()}_{create_config_files.config.daq_session_name}_connectivity-service.txt",
+            / f"log_{getpass.getuser()}_{create_config_files.integtest_params.daq_session_name}_connectivity-service.txt",
             "w",
         )
         connsvc_obj = subprocess.Popen(
-            f"gunicorn -b 0.0.0.0:{create_config_files.config.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(),
+            f"gunicorn -b 0.0.0.0:{create_config_files.integtest_params.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(),
             stdout=connsvc_log,
             stderr=connsvc_log,
             env=connsvc_env,
         )
 
-    elif create_config_files.config.connsvc_debug_level is not None:
-        set_session_env_var(str(create_config_files.config_file), create_config_files.config.config_session_name,
-                            "CONNECTION_FLASK_DEBUG", create_config_files.config.connsvc_debug_level, overwrite=True)
+    elif create_config_files.integtest_params.connsvc_debug_level is not None:
+        set_session_env_var(str(create_config_files.dunedaq_config_file), create_config_files.integtest_params.config_session_name,
+                            "CONNECTION_FLASK_DEBUG", create_config_files.integtest_params.connsvc_debug_level, overwrite=True)
 
     dunerc = request.config.getoption("--dunerc-path")
     if dunerc is None:
@@ -569,13 +569,13 @@ class RunResult:
         temp_suffix = ".temp_saved"
         now = time.time()
         for file_obj in rawdata_dir.glob(
-            f"{create_config_files.config.op_env}_raw*.hdf5"
+            f"{create_config_files.integtest_params.op_env}_raw*.hdf5"
         ):
             print(f"Renaming raw data file from earlier test: {str(file_obj)}")
             new_name = str(file_obj) + temp_suffix
             file_obj.rename(new_name)
         for file_obj in rawdata_dir.glob(
-            f"{create_config_files.config.op_env}_raw*.hdf5{temp_suffix}"
+            f"{create_config_files.integtest_params.op_env}_raw*.hdf5{temp_suffix}"
         ):
             modified_time = file_obj.stat().st_mtime
             if (now - modified_time) > 3600:
@@ -589,13 +589,13 @@ class RunResult:
         temp_suffix = ".temp_saved"
         now = time.time()
         for file_obj in tpset_dir.glob(
-            f"{create_config_files.config.op_env}_tp*.hdf5"
+            f"{create_config_files.integtest_params.op_env}_tp*.hdf5"
         ):
             print(f"Renaming TP data file from earlier test: {str(file_obj)}")
             new_name = str(file_obj) + temp_suffix
             file_obj.rename(new_name)
         for file_obj in tpset_dir.glob(
-            f"{create_config_files.config.op_env}_tp*.hdf5{temp_suffix}"
+            f"{create_config_files.integtest_params.op_env}_tp*.hdf5{temp_suffix}"
         ):
             modified_time = file_obj.stat().st_mtime
             if (now - modified_time) > 3600:
@@ -609,13 +609,13 @@ class RunResult:
         temp_suffix = ".temp_saved"
         now = time.time()
         for file_obj in trmon_dir.glob(
-            f"{create_config_files.config.op_env}_trmon*.hdf5"
+            f"{create_config_files.integtest_params.op_env}_trmon*.hdf5"
         ):
             print(f"Renaming TRMon data file from earlier test: {str(file_obj)}")
             new_name = str(file_obj) + temp_suffix
             file_obj.rename(new_name)
         for file_obj in trmon_dir.glob(
-            f"{create_config_files.config.op_env}_trmon*.hdf5{temp_suffix}"
+            f"{create_config_files.integtest_params.op_env}_trmon*.hdf5{temp_suffix}"
         ):
             modified_time = file_obj.stat().st_mtime
             if (now - modified_time) > 3600:
@@ -631,8 +631,8 @@ class RunResult:
     # 25-Mar-2026, KAB: use subprocess.Popen to manage the run control session so that we can
     # capture the console output and pass it back to the user for inspection and validation.
     popen_command_list = [dunerc] + dunerc_option_strings + [process_manager_type] \
-        + [str(create_config_files.config_file)] + [str(create_config_files.config.config_session_name)] \
-        + [str(create_config_files.config.daq_session_name)] + run_control_commands
+        + [str(create_config_files.dunedaq_config_file)] + [str(create_config_files.integtest_params.config_session_name)] \
+        + [str(create_config_files.integtest_params.daq_session_name)] + run_control_commands
     rc_process = subprocess.Popen(
         popen_command_list,
         stdout=subprocess.PIPE,
@@ -721,39 +721,40 @@ class RunResult:
             pass
         connsvc_obj.kill()
 
-    if create_config_files.config.attempt_cleanup:
+    if create_config_files.integtest_params.attempt_cleanup:
         print(
             "Checking for remaining gunicorn and drunc-controller processes", flush=True
         )
         subprocess.run(["killall", "gunicorn", "drunc-controller"])
 
-    result.confgen_config = create_config_files.config
-    result.config_session_name = create_config_files.config.config_session_name
-    result.daq_session_name = create_config_files.config.daq_session_name
+    result.confgen_config = create_config_files.integtest_params
+    result.config_session_name = create_config_files.integtest_params.config_session_name
+    result.daq_session_name = create_config_files.integtest_params.daq_session_name
     result.dunerc_commands = run_control_commands
     result.run_dir = run_dir
-    result.config_dir = create_config_files.config_dir
+    result.dunedaq_config_dir = create_config_files.dunedaq_config_dir
     result.data_files = []
     for rawdata_dir in rawdata_dirs:
         result.data_files += list(
-            rawdata_dir.glob(f"{create_config_files.config.op_env}_raw_*.hdf5")
+            rawdata_dir.glob(f"{create_config_files.integtest_params.op_env}_raw_*.hdf5")
         )
     result.tpset_files = []
     for tpset_dir in tpset_dirs:
         result.tpset_files += list(
-            tpset_dir.glob(f"{create_config_files.config.op_env}_tp_*.hdf5")
+            tpset_dir.glob(f"{create_config_files.integtest_params.op_env}_tp_*.hdf5")
         )
     result.trmon_files = []
     for trmon_dir in trmon_dirs:
         result.trmon_files += list(
-            trmon_dir.glob(f"{create_config_files.config.op_env}_trmon_*.hdf5")
+            trmon_dir.glob(f"{create_config_files.integtest_params.op_env}_trmon_*.hdf5")
         )
     result.log_files = list(run_dir.glob("log_*.txt")) + list(run_dir.glob("log_*.log"))
     result.opmon_files = list(run_dir.glob(f"info*{result.daq_session_name}*.json"))
     # 10-Dec-2025, KAB: added the DAQ session overall time so that we can use this
     # information in fine-tuning the allowed ranges in time-based checking of test results.
     result.daq_session_overall_time = time_after - time_before
     result.verbosity_helper = VerbosityHelper(integtest_verbosity_level)
+    result.user_requests_hdf5_file_removal = request.config.getoption("--remove-hdf5-files")
     if number_of_lines_printed_to_the_console > 0:
         print("---------- DRUNC Session END ----------", flush=True)
         print("", flush=True)

src/integrationtest/utility_functions.py

@@ -0,0 +1,82 @@
+import pytest
+import os
+import re
+from integrationtest.verbosity_helper import IntegtestVerbosityLevels
+
+import functools
+print = functools.partial(print, flush=True)  # always flush print() output
+
+def basic_checks(run_dunerc, caplog, print_test_name: bool = True):
+
+    # print out the name of the current test, if requested
+    if print_test_name and run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.drunc_transitions):
+        # print the name of the current test
+        current_test = os.environ.get("PYTEST_CURRENT_TEST")
+        match_obj = re.search(r".*\[(.+)-run_.*rc.*\d].*", current_test)
+        if match_obj:
+            current_test = match_obj.group(1)
+        banner_line = re.sub(".", "=", current_test)
+        print(banner_line)
+        print(current_test)
+        print(banner_line)
+
+    # Check that dunerc completed correctly
+    if run_dunerc.completed_process.returncode != 0:
+        fail_msg = f"The run control session returned a non-zero status code ({run_dunerc.completed_process.returncode})."
+        pytest.fail(fail_msg, pytrace=False)
+
+    # Check that there weren't any warnings or errors during setup
+    setup_logs = caplog.get_records("setup")
+    if len(setup_logs) > 0:
+        fail_msg = f"One or more problems were encountered during the setup of the pytest: {setup_logs}"
+        pytest.fail(fail_msg, pytrace=False)
+
+
+def remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal: bool = False):
+
+    # if the user requested that the files should be kept, we can simply exit early
+    if (run_dunerc.user_requests_hdf5_file_removal is not None and
+        "false" in run_dunerc.user_requests_hdf5_file_removal.lower()):
+        return
+
+    # if either of the integtest writer or the user running the test requested that the HDF5 files
+    # be deleted at the end of the test, do that.
+    if ((run_dunerc.user_requests_hdf5_file_removal is not None and
+         "true" in run_dunerc.user_requests_hdf5_file_removal.lower()) or
+        this_test_requests_hdf5_file_removal):
+        pathlist_string = ""
+        filelist_string = ""
+        for data_file in run_dunerc.data_files:
+            filelist_string += " " + str(data_file)
+            if str(data_file.parent) not in pathlist_string:
+                pathlist_string += " " + str(data_file.parent)
+        for data_file in run_dunerc.tpset_files:
+            filelist_string += " " + str(data_file)
+            if str(data_file.parent) not in pathlist_string:
+                pathlist_string += " " + str(data_file.parent)
+        for data_file in run_dunerc.trmon_files:
+            filelist_string += " " + str(data_file)
+            if str(data_file.parent) not in pathlist_string:
+                pathlist_string += " " + str(data_file.parent)
+
+        if pathlist_string and filelist_string:
+            if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
+                print("============================================")
+                print("Listing the hdf5 files before deleting them:")
+                print("============================================")
+
+                os.system(f"df -h {pathlist_string}")
+                print("--------------------")
+                os.system(f"ls -alF {filelist_string}")
+
+            for data_file in run_dunerc.data_files:
+                data_file.unlink()
+            for data_file in run_dunerc.tpset_files:
+                data_file.unlink()
+            for data_file in run_dunerc.trmon_files:
+                data_file.unlink()
+
+            if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
+                print("--------------------")
+                os.system(f"df -h {pathlist_string}")
+                print("============================================")