[SPARK-55487][PYTHON][TESTS] Detect dangling tests

dev/run-tests.py

@@ -27,6 +27,7 @@
 from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES
 from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which
 from sparktestsupport.utils import (
+    determine_dangling_python_tests,
     determine_modules_for_files,
     determine_modules_to_test,
     identify_changed_files_from_git_commits,
@@ -546,6 +547,14 @@ def main():
                     os.environ["GITHUB_SHA"], target_ref=os.environ["GITHUB_PREV_SHA"]
                 )
 
+            dangling_python_tests = determine_dangling_python_tests(changed_files)
+            if dangling_python_tests:
+                print(
+                    f"[error] Found the following dangling Python tests {', '.join(dangling_python_tests)}"
+                )
+                print("[error] Please add the tests to the appropriate module.")
+                sys.exit(1)
+
             modules_to_test = determine_modules_to_test(
                 determine_modules_for_files(changed_files), deduplicated=False
             )

dev/sparktestsupport/modules.py

@@ -19,6 +19,7 @@
 import itertools
 import os
 import re
+from pathlib import Path
 
 all_modules = []
 
@@ -89,6 +90,19 @@ def __init__(
     def contains_file(self, filename):
         return any(re.match(p, filename) for p in self.source_file_prefixes)
 
+    def missing_potential_python_test(self, filename):
+        """
+        Check if the given filename is missing from the module if it is a Python test file.
+
+        Return True if it is a test file and is not included in the module.
+        """
+        path = Path(filename)
+        last_part = path.parts[-1]
+        if not re.match(r"test_.*\.py", last_part):
+            return False
+        module_path = ".".join(path.parts)[:-3]  # Remove the ".py" suffix
+        return not any(module_path.endswith(test) for test in self.python_test_goals)
+
     def __repr__(self):
         return "Module<%s>" % self.name
 

dev/sparktestsupport/utils.py

@@ -159,6 +159,18 @@ def determine_modules_to_test(changed_modules, deduplicated=True):
     )
 
 
+def determine_dangling_python_tests(changed_files):
+    """
+    Given a list of changed files, return the set of Python tests that are not associated with any
+    module.
+    """
+    dangling_tests = set()
+    for filename in changed_files:
+        if modules.root.missing_potential_python_test(filename):
+            dangling_tests.add(filename)
+    return dangling_tests
+
+
 def determine_tags_to_exclude(changed_modules):
     tags = []
     for m in modules.all_modules: