feat: introduce timeout and retry parameter for BM executor shutdown

The awaitBinaryStorageExecutorTermination() in
MonitoredClusteringBuilderState is extended with parameters that allow
the timeout for the shutdown and the number of attempts to make to be
specified.

(also implement 2 small cleanups of code causing a compile error
depending on settings and a FB warning suppression that is no longer
necessary)

Author: ava-fred

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java

@@ -778,38 +778,87 @@ protected void deleteBinaryResources(final Set<URI> toBeDeleted) {
 
   /**
    * Waits until binary models are stored.
+   * Uses default parameters for timeout and retries, kept for backward compatibility.
    */
-  @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE")
   protected void awaitBinaryStorageExecutorTermination() {
-    LOGGER.info("Waiting for binary resource storage tasks to complete"); //$NON-NLS-1$
+    awaitBinaryStorageExecutorTermination(1, TimeUnit.MINUTES, 0);
+  }
+
+  /**
+   * Waits until binary models are stored. Waits for a given time and makes a given number of attempts.
+   *
+   * @param timeout
+   *          time to wait for shutdown
+   * @param unit
+   *          {@link TimeUnit} for timeout
+   * @param retryCount
+   *          number of retries to attempt
+   */
+  @SuppressWarnings("nls")
+  protected void awaitBinaryStorageExecutorTermination(final int timeout, final TimeUnit unit, final int retryCount) {
+    LOGGER.info("Waiting for binary resource storage tasks to complete: timeout {} {}, retryCount {}", timeout, unit, retryCount); //$NON-NLS-1$
     if (hwmTimeStamp != null) {
       LOGGER.info("high water mark was {} at {}", binaryStorageHighWaterMark, hwmTimeStamp); //$NON-NLS-1$
     }
 
     // Stop accepting additional work
     binaryStorageExecutor.shutdown();
-    long queuedTaskCount = binaryStorageExecutor.getQueue().size();
-    long activeTaskCount = binaryStorageExecutor.getActiveCount();
 
-    // Attempt to wait for queued work to complete
+    int retries = 0;
+    boolean terminated = false;
+    boolean stuck = false;
+
+    long prevQueuedTaskCount = binaryStorageExecutor.getQueue().size();
+    long prevActiveTaskCount = binaryStorageExecutor.getActiveCount();
+
     try {
-      if (!binaryStorageExecutor.awaitTermination(1, TimeUnit.MINUTES)) {
-        throw new InterruptedException();
+      do {
+        terminated = binaryStorageExecutor.awaitTermination(timeout, unit);
+        if (!terminated) {
+          // check whether if there is progress
+          long currQueuedTaskCount = binaryStorageExecutor.getQueue().size();
+          long currActiveTaskCount = binaryStorageExecutor.getActiveCount();
+
+          if (currQueuedTaskCount < prevQueuedTaskCount || currActiveTaskCount < prevActiveTaskCount) {
+            LOGGER.warn("Binary resource storage tasks not completed in time, start with {} queued / {} active; now have {} / {}", prevQueuedTaskCount, prevActiveTaskCount, currQueuedTaskCount, currActiveTaskCount);
+            if (retries < retryCount) {
+              retries += 1;
+              LOGGER.warn("retrying shutdown, attempt {} of {}", retries, retryCount);
+              prevQueuedTaskCount = currQueuedTaskCount;
+              prevActiveTaskCount = currActiveTaskCount;
+            }
+          } else {
+            LOGGER.warn("Binary resource storage tasks not completed in time, not making progress, stuck on {} / {} queued / active tasks", currQueuedTaskCount, currActiveTaskCount);
+            stuck = true;
+          }
+        }
+
+      } while (!terminated && !stuck && retries < retryCount);
+
+      if (terminated) {
+        LOGGER.info("Binary resource storage executor completed.");
+      } else {
+        LOGGER.warn("Binary resource storage executor shutdown not successful, terminating");
+        terminateBinaryStorageExecutor();
       }
-    } catch (InterruptedException e) { // NOPMD ExceptionAsFlowControl
-      LOGGER.warn(String.format("Binary resource storage tasks not completed in time, start with %d queued / %d active; now have %d / %d", //$NON-NLS-1$
-          queuedTaskCount, activeTaskCount, binaryStorageExecutor.getQueue().size(), binaryStorageExecutor.getActiveCount()));
-      binaryStorageExecutor.shutdownNow();
+    } catch (InterruptedException e) {
+      LOGGER.warn("Interrupted waiting for binaryStorageExecutor shutdown, terminating");
+      terminateBinaryStorageExecutor();
     }
 
-    LOGGER.info("Binary resource storage executor completed."); //$NON-NLS-1$
-
     // Be ready to accept additional work
     binaryStorageExecutor = makeBinaryStorageExecutor();
     binaryStorageHighWaterMark = 0;
     hwmTimeStamp = null;
   }
 
+  @SuppressWarnings("nls")
+  private void terminateBinaryStorageExecutor() {
+    LOGGER.warn("Terminating binaryStorageExecutor");
+    List<Runnable> tasks = binaryStorageExecutor.shutdownNow();
+    LOGGER.warn("{} tasks not processed", tasks.size());
+  }
+
   /**
    * Updates the markers on a single resource.
    *

com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/util/ParseTreeUtil.java

@@ -138,8 +138,8 @@ public static String getParsedString(final EObject object, final String featureN
   public static void ensureNodeModelLoaded(final Resource resource) {
     if (resource instanceof LazyLinkingResource2 lazyLinkinResource && lazyLinkinResource.isLoadedFromStorage()) {
       EObject root = resource.getContents().get(0);
-      if (root != null && NodeModelUtils.getNode(root) instanceof ICompositeNode composite) {
-        composite.getText(); // side-effect on removing com.avaloq.tools.ddk.xtext.resource.persistence.ProxyCompositeNode
+      if (root != null) {
+        NodeModelUtils.getNode(root).getText(); // side-effect on removing com.avaloq.tools.ddk.xtext.resource.persistence.ProxyCompositeNode
       }
     }
   }