Improve exit handling of O2HitMerger

So far, the exit status of O2HitMerger was not analysed.
This could lead to situations where O2HitMerger was killed by the OS
due to out-of-memory, yet the o2-sim simulator still exited as "successfull".

This commit improves the handling. Problems in O2HitMerger will lead
to exit code 1 of o2-sim.

Author: sawenzel

run/O2HitMerger.h

@@ -87,15 +87,6 @@ namespace o2
 namespace devices
 {
 
-// signal handler
-void sighandler(int signal)
-{
-  if (signal == SIGSEGV) {
-    LOG(warn) << "segmentation violation ... just exit without coredump in order not to hang";
-    raise(SIGKILL);
-  }
-}
-
 class O2HitMerger : public fair::mq::Device
 {
 
@@ -130,7 +121,6 @@ class O2HitMerger : public fair::mq::Device
   void InitTask() final
   {
     LOG(info) << "INIT HIT MERGER";
-    // signal(SIGSEGV, sighandler);
     ROOT::EnableThreadSafety();
 
     std::string outfilename("o2sim_merged_hits.root"); // default name
@@ -764,7 +754,6 @@ class O2HitMerger : public fair::mq::Device
         eventheader->putInfo("prims_eta_0.8_pi", eta0Point8CounterPi);
         eventheader->putInfo("prims_total", prims);
       };
-
       reorderAndMergeMCTracks(flusheventID, mOutTree, nprimaries, subevOrdered, mcheaderhook, eventheader);
 
       if (mOutTree) {

run/o2sim_parallel.cxx

@@ -732,27 +732,51 @@ int main(int argc, char* argv[])
   int status, cpid;
   // wait just blocks and waits until any child returns; but we make sure to wait until merger is here
   bool errored = false;
-  while ((cpid = wait(&status)) != mergerpid) {
+  // wait at least until mergerpid is reaped
+  while ((cpid = wait(&status)) != -1) {
+    if (cpid == mergerpid) {
+      break; // Defer handling of mergerpid exit status until after the loop
+    }
+
     if (WEXITSTATUS(status) || WIFSIGNALED(status)) {
       if (!shutdown_initiated) {
-        LOG(info) << "Process " << cpid << " EXITED WITH CODE " << WEXITSTATUS(status) << " SIGNALED "
-                  << WIFSIGNALED(status) << " SIGNAL " << WTERMSIG(status);
+        LOG(info) << "Process " << cpid << " EXITED WITH CODE " << WEXITSTATUS(status)
+                  << " SIGNALED " << WIFSIGNALED(status) << " SIGNAL " << WTERMSIG(status);
 
-        // we bring down all processes if one of them had problems or got a termination signal
-        // if (WTERMSIG(status) == SIGABRT || WTERMSIG(status) == SIGSEGV || WTERMSIG(status) == SIGBUS || WTERMSIG(status) == SIGTERM) {
         LOG(info) << "Problem detected (or child received termination signal) ... shutting down whole system ";
         for (auto p : gChildProcesses) {
           LOG(info) << "TERMINATING " << p;
           if (killpg(p, 0) == 0) {
-            killpg(p, SIGTERM); // <--- makes sure to shutdown "unknown" child pids via the group property
+            killpg(p, SIGTERM);
           }
         }
         LOG(error) << "SHUTTING DOWN DUE TO SIGNALED EXIT IN COMPONENT " << cpid;
-        o2::simpubsub::publishMessage(externalpublishchannel, o2::simpubsub::simStatusString("O2SIM", "STATE", "FAILURE"));
+        o2::simpubsub::publishMessage(
+          externalpublishchannel,
+          o2::simpubsub::simStatusString("O2SIM", "STATE", "FAILURE"));
+        errored = true;
+      }
+    }
+  }
+  // Handle mergerpid status separately
+  if (cpid == mergerpid) {
+    if (WIFEXITED(status)) {
+      // anything other than 128 is indicative of error
+      if (WEXITSTATUS(status) != 128) {
+        LOG(error) << "Merger process exited with abnormal code " << WEXITSTATUS(status);
+        errored = true;
+      }
+    } else if (WIFSIGNALED(status)) {
+      auto sig = WTERMSIG(status);
+      if (sig == SIGKILL || sig == SIGBUS || sig == SIGSEGV || sig == SIGABRT) {
+        LOG(error) << "Merger process terminated through abnormal signal " << WTERMSIG(status);
         errored = true;
       }
+    } else {
+      LOG(warning) << "Merger process exited with unexpected status.";
     }
   }
+
   // This marks the actual end of the computation (since results are available)
   LOG(info) << "Merger process " << mergerpid << " returned";
   LOG(info) << "Simulation process took " << timer.RealTime() << " s";