Improve test diagnostics when expected_results.txt contains syntax errors

tests/utils/stl/test/file_parsing.py

@@ -101,7 +101,7 @@ def _do_crosslist(ctx: _ParseCtx):
 
 
 def _parse_env_lst(env_lst: Path, ctx: _ParseCtx):
-    for line in parse_commented_file(env_lst):
+    for _, line in _parse_commented_file(env_lst):
         if (m:=_INCLUDE_REGEX.match(line)) is not None:
             p = env_lst.parent / Path(m.group("filename"))
             _parse_env_lst(p, ctx)
@@ -112,19 +112,19 @@ def _parse_env_lst(env_lst: Path, ctx: _ParseCtx):
             ctx.current.append(_parse_env_line(line))
 
 
-def parse_commented_file(filename: Union[str, bytes, os.PathLike]) \
-        -> List[str]:
+def _parse_commented_file(filename: Union[str, bytes, os.PathLike]) \
+        -> List[Tuple[int, str]]:
     if str(filename) in _preprocessed_file_cache:
         return _preprocessed_file_cache[str(filename)]
 
     filename_path = Path(filename)
     result = list()
     with filename_path.open() as f:
-        for line in f.readlines():
+        for line_number, line in enumerate(f.readlines(), start=1):
             if (line:=_COMMENT_REGEX.sub("", line)):
                 line = line.strip()
                 if line:
-                    result.append(line)
+                    result.append((line_number, line))
 
         _preprocessed_file_cache[str(filename)] = result
         return result
@@ -136,17 +136,17 @@ def parse_result_file(filename: Union[str, bytes, os.PathLike]) \
         return _expected_result_entry_cache[str(filename)]
 
     res = dict()
-    for line in parse_commented_file(filename):
+    for line_number, line in _parse_commented_file(filename):
         m = _EXPECTED_RESULT_REGEX.match(line)
         if m is None:
-            raise Exception(f'Incorrectly formatted line "{line}" in {filename}.')
+            raise Exception(f'Incorrectly formatted line {line_number}: "{line}" in {filename}.')
         prefix = m.group("prefix")
         result = m.group("result")
         result_code = getattr(lit.Test, result, None)
         if result_code is None:
             result_code = getattr(stl.test.tests, result, None)
         if result_code is None:
-            raise Exception(f'Unknown result code "{result}" in line "{line}" in {filename}.')
+            raise Exception(f'Unknown result code "{result}" in line {line_number}: "{line}" in {filename}.')
         res[prefix] = result_code
 
     _expected_result_entry_cache[str(filename)] = res

tools/scripts/check_libcxx_paths.py

@@ -21,16 +21,49 @@
         stripped_lines = [line.strip() for line in file]
 
     filtered_lines = [s for s in stripped_lines if s and not s.startswith("#")] # Ignore empty lines and comments.
-    config_rgx = re.compile(r"(:\d+)? (FAIL|SKIPPED)$")
-    unique_tests = {config_rgx.sub("", line) for line in filtered_lines}
+
+    damaged_lines = set()
+    unknown_results = set()
+    unique_tests = set()
+
+    syntax_rgx = re.compile(r"(?P<test>[^: ]+)(?P<config>:\d+)? (?P<result>\w+)")
+    for line in filtered_lines:
+        m = syntax_rgx.fullmatch(line)
+        if m is None:
+            damaged_lines.add(line)
+        else:
+            result = m.group("result")
+            if result == "FAIL" or result == "SKIPPED":
+                unique_tests.add(m.group("test"))
+            else:
+                unknown_results.add(line)
 
     # Build up a list of nonexistent tests so they can be printed in sorted order.
     absolute_libcxx_test = absolute_repo_path / "llvm-project/libcxx/test"
     nonexistent_tests = [s for s in unique_tests if not (absolute_libcxx_test / s).is_file()]
 
+    failed = False
+
+    if damaged_lines:
+        failed = True
+        print(f"Failure: {expected_results_txt} contains {len(damaged_lines)} damaged lines:", file=sys.stderr)
+        print(f"(Double-check their syntax. Did you forget the result code at the end?)", file=sys.stderr)
+        for s in sorted(damaged_lines):
+            print(f"{s}", file=sys.stderr)
+
+    if unknown_results:
+        failed = True
+        print(f"Failure: {expected_results_txt} contains {len(unknown_results)} unknown results:", file=sys.stderr)
+        print(f"(Only FAIL and SKIPPED are known.)", file=sys.stderr)
+        for s in sorted(unknown_results):
+            print(f"{s}", file=sys.stderr)
+
     if nonexistent_tests:
+        failed = True
         print(f"Failure: {expected_results_txt} contains {len(nonexistent_tests)} nonexistent tests:", file=sys.stderr)
         for s in sorted(nonexistent_tests):
             print(f"{s}", file=sys.stderr)
+
+    if failed:
         print(f"##vso[task.logissue type=error]{expected_results_txt} failed validation, see log.", file=sys.stderr)
         sys.exit(1)