Fix deadlock when the compiler crashes internally

pyop2/compilation.py

@@ -346,7 +346,7 @@ class LinuxGnuCompiler(Compiler):
     _cxxflags = ("-fPIC", "-Wall")
     _ldflags = ("-shared",)
 
-    _optflags = ("-march=native", "-O3", "-ffast-math")
+    _optflags = ("-march=native", "-O3")
     _debugflags = ("-O0", "-g")
 
     @property
@@ -543,15 +543,14 @@ def check_source_hashes(compiler, code, extension, comm):
     make_cache=lambda: CompilerDiskAccess(configuration['cache_dir'], extension=".so")
 )
 @PETSc.Log.EventDecorator()
-def make_so(compiler, code, extension, comm, filename=None):
+def make_so(compiler, code, extension, comm):
     """Build a shared library and load it
 
     :arg compiler: The compiler to use to create the shared library.
     :arg code: The code to compile.
     :arg filename: The filename of the library to create.
     :arg extension: extension of the source file (c, cpp).
     :arg comm: Communicator over which to perform compilation.
-    :arg filename: Optional
     Returns a :class:`ctypes.CDLL` object of the resulting shared
     library."""
     # Compilation communicators are reference counted on the PyOP2 comm
@@ -567,69 +566,95 @@ def make_so(compiler, code, extension, comm, filename=None):
         compiler_flags = compiler.cflags
 
     # Compile on compilation communicator (ccomm) rank 0
-    soname = None
     if ccomm.rank == 0:
-        if filename is None:
+        try:
             # Adding random 2-digit hexnum avoids using excessive filesystem inodes
             tempdir = MEM_TMP_DIR.joinpath(f"{randint(0, 255):02x}")
             tempdir.mkdir(parents=True, exist_ok=True)
             # This path + filename should be unique
             descriptor, filename = mkstemp(suffix=f".{extension}", dir=tempdir, text=True)
             filename = Path(filename)
-        else:
-            filename.parent.mkdir(exist_ok=True)
-
-        cname = filename
-        oname = filename.with_suffix(".o")
-        soname = filename.with_suffix(".so")
-        logfile = filename.with_suffix(".log")
-        errfile = filename.with_suffix(".err")
-        with progress(INFO, 'Compiling wrapper'):
-            # Write source code to disk
-            with open(cname, "w") as fh:
-                fh.write(code)
-            os.close(descriptor)
 
-            if not compiler.ld:
-                # Compile and link
-                cc = (exe,) + compiler_flags + ('-o', str(soname), str(cname)) + compiler.ldflags
-                _run(cc, logfile, errfile)
-            else:
-                # Compile
-                cc = (exe,) + compiler_flags + ('-c', '-o', str(oname), str(cname))
-                _run(cc, logfile, errfile)
-                # Extract linker specific "cflags" from ldflags and link
-                ld = tuple(shlex.split(compiler.ld)) + ('-o', str(soname), str(oname)) + tuple(expandWl(compiler.ldflags))
-                _run(ld, logfile, errfile, step="Linker", filemode="a")
+            cname = filename
+            oname = filename.with_suffix(".o")
+            soname = filename.with_suffix(".so")
+            logfile = filename.with_suffix(".log")
+            errfile = filename.with_suffix(".err")
+        except BaseException as e:
+            result = e
+        else:
+            try:
+                with progress(INFO, 'Compiling wrapper'):
+                    # Write source code to disk
+                    with open(cname, "w") as fh:
+                        fh.write(code)
+                    os.close(descriptor)
+
+                    if not compiler.ld:
+                        # Compile and link
+                        cc = (exe,) + compiler_flags + ('-o', str(soname), str(cname)) + compiler.ldflags
+                        _run(cc, logfile, errfile)
+                    else:
+                        # Compile
+                        cc = (exe,) + compiler_flags + ('-c', '-o', str(oname), str(cname))
+                        _run(cc, logfile, errfile)
+                        # Extract linker specific "cflags" from ldflags and link
+                        ld = tuple(shlex.split(compiler.ld)) + ('-o', str(soname), str(oname)) + tuple(expandWl(compiler.ldflags))
+                        _run(ld, logfile, errfile, step="Linker", filemode="a")
+
+                result = soname
+            except subprocess.CalledProcessError as e:
+                msg = dedent(f"""
+                    Command "{e.cmd}" return error status {e.returncode}.
+                    Unable to compile code
+                """)
+                if os.environ.get("FIREDRAKE_CI", False):
+                    msg += dedent(f"""
+                        Code is:
+                        {code}
+                    """)
+                    with open(errfile) as err:
+                        msg += dedent(f"""
+                            Compiler output is:
+                            {''.join(err.readlines())}
+                        """)
+                else:
+                    msg += dedent(f"""
+                        Compile log in {logfile!s}
+                        Compile errors in {errfile!s}
+                    """)
+                result = CompilationError(msg)
+                result.__cause__ = e  # equivalent to 'raise XXX from e'
+            except BaseException as e:
+                # catch and broadcast all exceptions to prevent deadlocks
+                result = e
+    else:
+        result = None
 
-    return ccomm.bcast(soname, root=0)
+    result = ccomm.bcast(result)
+    if isinstance(result, Exception):
+        raise result
+    else:
+        return result
 
 
 def _run(cc, logfile, errfile, step="Compilation", filemode="w"):
     """ Run a compilation command and handle logging + errors.
     """
     debug(f"{step} command: {' '.join(cc)}")
-    try:
-        if configuration['no_fork_available']:
-            redirect = ">" if filemode == "w" else ">>"
-            cc += (f"2{redirect}", str(errfile), redirect, str(logfile))
-            cmd = " ".join(cc)
-            status = os.system(cmd)
-            if status != 0:
-                raise subprocess.CalledProcessError(status, cmd)
-        else:
-            with open(logfile, filemode) as log, open(errfile, filemode) as err:
-                log.write(f"{step} command:\n")
-                log.write(" ".join(cc))
-                log.write("\n\n")
-                subprocess.check_call(cc, stderr=err, stdout=log)
-    except subprocess.CalledProcessError as e:
-        raise CompilationError(dedent(f"""
-            Command "{e.cmd}" return error status {e.returncode}.
-            Unable to compile code
-            Compile log in {logfile!s}
-            Compile errors in {errfile!s}
-            """))
+    if configuration['no_fork_available']:
+        redirect = ">" if filemode == "w" else ">>"
+        cc += (f"2{redirect}", str(errfile), redirect, str(logfile))
+        cmd = " ".join(cc)
+        status = os.system(cmd)
+        if status != 0:
+            raise subprocess.CalledProcessError(status, cmd)
+    else:
+        with open(logfile, filemode) as log, open(errfile, filemode) as err:
+            log.write(f"{step} command:\n")
+            log.write(" ".join(cc))
+            log.write("\n\n")
+            subprocess.check_call(cc, stderr=err, stdout=log)
 
 
 def add_profiling_events(dll, events):

tests/firedrake/output/test_io_backward_compat.py

@@ -1,5 +1,4 @@
 import pytest
-import os
 from os.path import abspath, dirname, join, exists
 from firedrake import *
 from firedrake.mesh import make_mesh_from_coordinates
@@ -275,8 +274,6 @@ def test_io_backward_compat_timestepping_save(version):
 @pytest.mark.parallel(nprocs=4)
 @pytest.mark.parametrize('version', ["2024_01_27"])
 def test_io_backward_compat_timestepping_load(version):
-    if os.getenv("FIREDRAKE_CI") == "1":
-        pytest.skip("Skipping on CI to avoid unidentified timeout")
     filename = join(filedir, "_".join([basename, version, "timestepping" + ".h5"]))
     with CheckpointFile(filename, "r") as afile:
         mesh = afile.load_mesh(mesh_name)
@@ -292,8 +289,6 @@ def test_io_backward_compat_timestepping_load(version):
 @pytest.mark.parallel(nprocs=3)
 @pytest.mark.parametrize('version', ["2024_01_27"])
 def test_io_backward_compat_timestepping_append(version, tmpdir):
-    if os.getenv("FIREDRAKE_CI") == "1":
-        pytest.skip("Skipping on CI to avoid unidentified timeout")
     filename = join(filedir, "_".join([basename, version, "timestepping" + ".h5"]))
     copyname = join(str(tmpdir), "test_io_backward_compat_timestepping_append_dump.h5")
     copyname = COMM_WORLD.bcast(copyname, root=0)

tests/pyop2/test_caching.py

@@ -812,7 +812,7 @@ def test_writing_large_so():
             #include <stdio.h>\n
             void big(double *result){
             """)
-        variables = (f"v{next(tempfile._get_candidate_names())}" for _ in range(128*1024))
+        variables = (f"v{next(tempfile._get_candidate_names())}" for _ in range(1024))
         lines = (f"  double {v} = {hash(v)/1000000000};\n  *result += {v};\n" for v in variables)
         program = "\n".join(chain.from_iterable(((preamble, ), lines, ("}\n", ))))
         with open("big.c", "w") as fh: