Better detection of available system resources for decompilation and allow certain tasks to be marked such that nothing else runs concurrently

Author: lukebemish

build.gradle

@@ -254,9 +254,11 @@ dependencies {
         }
     }
     modelApi libs.gson
+    modelImplementation libs.oshi.core
     modelCompileOnlyApi cLibs.jspecify
     modelCompileOnly cLibs.bundles.compileonly
     modelAnnotationProcessor cLibs.bundles.annotationprocessor
+    modelApi libs.forkedtaskexecutor
 
     signaturesApi libs.asm
     signaturesCompileOnly cLibs.bundles.compileonly
@@ -276,7 +278,6 @@ dependencies {
             requireFeature('runner')
         }
     }
-    modelApi libs.forkedtaskexecutor
 
     instrumentationImplementation libs.asm
 }

gradle/libs.versions.toml

@@ -7,6 +7,7 @@ asm = "9.7"
 mappingio = "0.7.0"
 feather = "1.1.0"
 gson = "2.11.0"
+oshi = "6.6.5"
 
 [libraries]
 
@@ -24,3 +25,5 @@ feather-core = { group = "org.parchmentmc", name = "feather", version.ref = "fea
 feather-gson = { group = "org.parchmentmc.feather", name = "io-gson", version.ref = "feather" }
 
 gson = { group = "com.google.code.gson", name = "gson", version.ref = "gson" }
+
+oshi-core = { group = "com.github.oshi", name = "oshi-core", version.ref = "oshi" }

settings.gradle

@@ -26,6 +26,10 @@ dependencyResolutionManagement {
                 includeGroupAndSubgroups 'org.parchmentmc'
             }
         }
+        maven {
+            name 'Staging'
+            url 'https://maven.lukebemish.dev/staging/'
+        }
     }
 
     repositoriesMode = RepositoriesMode.FAIL_ON_PROJECT_REPOS

src/main/java/dev/lukebemish/taskgraphrunner/runtime/Task.java

@@ -322,10 +322,12 @@ private void execute(Context context) {
             // Something was not up-to-date -- so we run everything
             LOGGER.info("Starting task `{}`.", name);
             try {
-                if (parallelism == null) {
-                    run(context);
-                } else {
-                    context.lockManager().enforcedParallelism(context, parallelism, () -> run(context));
+                try (var ignored = LockManager.heavyLightLock(parallelism)) {
+                    if (parallelism == null) {
+                        run(context);
+                    } else {
+                        context.lockManager().enforcedParallelism(context, parallelism, () -> run(context));
+                    }
                 }
                 boolean nothingChanged = true;
                 for (var output : outputTypes().keySet()) {

src/main/java/dev/lukebemish/taskgraphrunner/runtime/util/LockManager.java

@@ -37,6 +37,8 @@ public class LockManager {
 
     private static final Map<String, String> parallelismGroups;
 
+    private static final Semaphore heavyLock = new Semaphore(Integer.MAX_VALUE, true);
+
     static {
         String groupsProperty = System.getProperty("dev.lukebemish.taskgraphrunner.parallelism.groups", "");
         Map<String, String> map = new LinkedHashMap<>();
@@ -53,6 +55,29 @@ public class LockManager {
         parallelismGroups = map;
     }
 
+    public static boolean isHeavy(String key) {
+        if (parallelismGroups.containsKey(key)) {
+            if (isHeavy(parallelismGroups.get(key))) {
+                return true;
+            }
+        }
+        return Boolean.getBoolean("dev.lukebemish.taskgraphrunner.parallelism."+key+".heavy");
+    }
+
+    public static LockLike heavyLightLock(String key) {
+        try {
+            if (isHeavy(key)) {
+                heavyLock.acquire(Integer.MAX_VALUE);
+                return new HeavyLock(heavyLock);
+            } else {
+                heavyLock.acquire();
+                return new LightLock(heavyLock);
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private static int findParallelism(String key) {
         if (parallelismGroups.containsKey(key)) {
             int maxInt = 0;
@@ -269,6 +294,30 @@ public sealed interface LockLike extends AutoCloseable {
         void close();
     }
 
+    public static final class LightLock implements LockLike {
+        private final Semaphore semaphore;
+        private LightLock(Semaphore semaphore) {
+            this.semaphore = semaphore;
+        }
+
+        @Override
+        public void close() {
+            semaphore.release();
+        }
+    }
+
+    public static final class HeavyLock implements LockLike {
+        private final Semaphore semaphore;
+        private HeavyLock(Semaphore semaphore) {
+            this.semaphore = semaphore;
+        }
+
+        @Override
+        public void close() {
+            semaphore.release(Integer.MAX_VALUE);
+        }
+    }
+
     public static final class Locks implements LockLike {
         private final List<? extends LockLike> locks;
 

src/model/java/dev/lukebemish/taskgraphrunner/model/conversion/NeoFormGenerator.java

@@ -339,7 +339,7 @@ public static Config convert(Path neoFormConfig, Value selfReference, Options op
                         args = toolModel.args;
                     }
                     if (isVineflower(function) && function.args().stream().noneMatch(s -> s.startsWith("-thr="))) {
-                        args.add(processArgument(downloadInputs, "-thr="+Runtime.getRuntime().availableProcessors(), step, source, listLibrariesName));
+                        args.add(processArgument(downloadInputs, "-thr="+SystemSpecsFinder.recommendedThreads(), step, source, listLibrariesName));
                     }
                     for (var arg : function.args()) {
                         args.add(processArgument(downloadInputs, arg, step, source, listLibrariesName));
@@ -552,6 +552,9 @@ private static Argument processArgument(Map<String, Input> inputs, String arg, N
             if (arg.startsWith("-Xmx")) {
                 var rest = arg.substring(4);
                 var systemProp = "dev.lukebemish.taskgraphrunner."+step.type()+".maxHeap";
+                if (isVineflower(function)) {
+                    rest = String.valueOf(SystemSpecsFinder.recommendedMemory());
+                }
                 return new Argument.Untracked("-Xmx{}", new InputValue.DirectInput(new Value.SystemPropertyValue(systemProp, rest)));
             } else if (arg.startsWith("-Xms")) {
                 var rest = arg.substring(4);

src/model/java/dev/lukebemish/taskgraphrunner/model/conversion/SingleVersionGenerator.java

@@ -341,10 +341,10 @@ public static Config convert(String version, Options options) throws IOException
         var decompileTask = new TaskModel.Tool(
             "decompile",
             List.of(
-                new Argument.Untracked("-Xmx{}", new InputValue.DirectInput(new Value.SystemPropertyValue("dev.lukebemish.taskgraphrunner.decompile.maxHeap", "4G"))),
+                new Argument.Untracked("-Xmx{}", new InputValue.DirectInput(new Value.SystemPropertyValue("dev.lukebemish.taskgraphrunner.decompile.maxHeap", String.valueOf(SystemSpecsFinder.recommendedMemory())))),
                 Argument.direct("-jar"),
                 new Argument.FileInput(null, new Input.DirectInput(Value.tool("vineflower")), PathSensitivity.NONE),
-                new Argument.Untracked("-thr={}", new InputValue.DirectInput(new Value.SystemPropertyValue("dev.lukebemish.taskgraphrunner.decompile.maxThreads", String.valueOf(Runtime.getRuntime().availableProcessors())))),
+                new Argument.Untracked("-thr={}", new InputValue.DirectInput(new Value.SystemPropertyValue("dev.lukebemish.taskgraphrunner.decompile.maxThreads", String.valueOf(SystemSpecsFinder.recommendedThreads())))),
                 Argument.direct("--decompile-inner"),
                 Argument.direct("--remove-bridge"),
                 Argument.direct("--decompile-generics"),

src/model/java/dev/lukebemish/taskgraphrunner/model/conversion/SystemSpecsFinder.java

@@ -0,0 +1,23 @@
+package dev.lukebemish.taskgraphrunner.model.conversion;
+
+import oshi.SystemInfo;
+
+class SystemSpecsFinder {
+    static long recommendedThreads() {
+        return Runtime.getRuntime().availableProcessors() / 2;
+    }
+
+    static long recommendedMemory() {
+        return Math.min(TOTAL_MEMORY / 5, GIGABYTE * 4);
+    }
+
+    private static final long TOTAL_MEMORY;
+
+    private static final long GIGABYTE = 1024 * 1024 * 1024;
+
+    static {
+        var systemInfo = new SystemInfo();
+        var hardware = systemInfo.getHardware();
+        TOTAL_MEMORY = hardware.getMemory().getTotal();
+    }
+}

src/model/java/module-info.java

@@ -2,6 +2,7 @@
     requires com.google.gson;
     requires static org.jspecify;
     requires dev.lukebemish.forkedtaskexecutor;
+    requires com.github.oshi;
 
     opens dev.lukebemish.taskgraphrunner.model.conversion to com.google.gson;
     opens dev.lukebemish.taskgraphrunner.model to com.google.gson;