Modified the integtests to support user-requested deletion of HDF5 files at the end of each integtest.

integtest/disabled_output_test.py

@@ -7,9 +7,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -171,7 +171,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=True)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=True)
 
 
 def test_log_files(run_dunerc):
@@ -220,3 +220,7 @@ def test_data_files(run_dunerc):
                 data_file, fragment_check_list[jdx]
             )
     assert all_ok
+
+
+def test_cleanup(run_dunerc):
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False)

integtest/hdf5_compression_test.py

@@ -5,9 +5,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -241,7 +241,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -304,33 +304,4 @@ def test_tpstream_files(run_dunerc):
 
 
 def test_cleanup(run_dunerc):
-    pathlist_string = ""
-    filelist_string = ""
-    for data_file in run_dunerc.data_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-    for data_file in run_dunerc.tpset_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-
-    if pathlist_string and filelist_string:
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("============================================")
-            print("Listing the hdf5 files before deleting them:")
-            print("============================================")
-
-            os.system(f"df -h {pathlist_string}")
-            print("--------------------")
-            os.system(f"ls -alF {filelist_string}")
-
-        for data_file in run_dunerc.data_files:
-            data_file.unlink()
-        for data_file in run_dunerc.tpset_files:
-            data_file.unlink()
-
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("--------------------")
-            os.system(f"df -h {pathlist_string}")
-            print("============================================")
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True)

integtest/insufficient_disk_space_test.py

@@ -5,9 +5,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -154,7 +154,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -205,27 +205,4 @@ def test_data_files(run_dunerc):
 
 
 def test_cleanup(run_dunerc):
-    pathlist_string = ""
-    filelist_string = ""
-    for data_file in run_dunerc.data_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-
-    if pathlist_string and filelist_string:
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("============================================")
-            print("Listing the hdf5 files before deleting them:")
-            print("============================================")
-
-            os.system(f"df -h {pathlist_string}")
-            print("--------------------")
-            os.system(f"ls -alF {filelist_string}")
-
-        for data_file in run_dunerc.data_files:
-            data_file.unlink()
-
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("--------------------")
-            os.system(f"df -h {pathlist_string}")
-            print("============================================")
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True)

integtest/large_trigger_record_test.py

@@ -12,9 +12,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -167,7 +167,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=True)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=True)
 
 
 def test_log_files(run_dunerc):
@@ -222,27 +222,4 @@ def test_data_files(run_dunerc):
 
 
 def test_cleanup(run_dunerc):
-    pathlist_string = ""
-    filelist_string = ""
-    for data_file in run_dunerc.data_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-
-    if pathlist_string and filelist_string:
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("============================================")
-            print("Listing the hdf5 files before deleting them:")
-            print("============================================")
-
-            os.system(f"df -h {pathlist_string}")
-            print("--------------------")
-            os.system(f"ls -alF {filelist_string}")
-
-        for data_file in run_dunerc.data_files:
-            data_file.unlink()
-
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("--------------------")
-            os.system(f"df -h {pathlist_string}")
-            print("============================================")
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True)

integtest/max_file_size_test.py

@@ -5,9 +5,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -210,7 +210,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -273,33 +273,4 @@ def test_tpstream_files(run_dunerc):
 
 
 def test_cleanup(run_dunerc):
-    pathlist_string = ""
-    filelist_string = ""
-    for data_file in run_dunerc.data_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-    for data_file in run_dunerc.tpset_files:
-        filelist_string += " " + str(data_file)
-        if str(data_file.parent) not in pathlist_string:
-            pathlist_string += " " + str(data_file.parent)
-
-    if pathlist_string and filelist_string:
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("============================================")
-            print("Listing the hdf5 files before deleting them:")
-            print("============================================")
-
-            os.system(f"df -h {pathlist_string}")
-            print("--------------------")
-            os.system(f"ls -alF {filelist_string}")
-
-        for data_file in run_dunerc.data_files:
-            data_file.unlink()
-        for data_file in run_dunerc.tpset_files:
-            data_file.unlink()
-
-        if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
-            print("--------------------")
-            os.system(f"df -h {pathlist_string}")
-            print("============================================")
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=True)

integtest/multiple_data_writers_test.py

@@ -5,9 +5,9 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
 import integrationtest.resource_validation as resource_validation
+import integrationtest.utility_functions as utility_functions
 from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
@@ -142,7 +142,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -180,3 +180,7 @@ def test_data_files(run_dunerc):
                 data_file, fragment_check_list[jdx]
             )
     assert all_ok, "\N{POLICE CARS REVOLVING LIGHT} One or more raw data file checks failed! \N{POLICE CARS REVOLVING LIGHT}"
+
+
+def test_cleanup(run_dunerc):
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False)

integtest/offline_prod_run_test.py

@@ -5,8 +5,8 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
+import integrationtest.utility_functions as utility_functions
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
 import functools
@@ -92,7 +92,7 @@
 
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -127,3 +127,7 @@ def test_data_files(run_dunerc):
                 data_file, fragment_check_list[jdx]
             )
     assert all_ok
+
+
+def test_cleanup(run_dunerc):
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False)

integtest/trmonrequestor_test.py

@@ -5,8 +5,8 @@
 
 import integrationtest.data_file_checks as data_file_checks
 import integrationtest.log_file_checks as log_file_checks
-import integrationtest.basic_checks as basic_checks
 import integrationtest.data_classes as data_classes
+import integrationtest.utility_functions as utility_functions
 from integrationtest.verbosity_helper import IntegtestVerbosityLevels
 
 import functools
@@ -105,7 +105,7 @@ def make_run_command_list(runnum):
 # The tests themselves
 def test_dunerc_success(run_dunerc, caplog):
     # checks for run control success, problems during pytest setup, etc.
-    basic_checks.basic_checks(run_dunerc, caplog, print_test_name=False)
+    utility_functions.basic_checks(run_dunerc, caplog, print_test_name=False)
 
 
 def test_log_files(run_dunerc):
@@ -205,3 +205,7 @@ def test_data_files(run_dunerc):
 
     assert all_ok
     assert trmon_ok
+
+
+def test_cleanup(run_dunerc):
+    utility_functions.remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal=False)