Run the compiler test in parallel

.vscode/launch.json

@@ -4,6 +4,14 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "type": "java",
+            "name": "BreakTest",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.compiler.BreakTest",
+            "projectName": "rascal",
+            "console": "integratedTerminal"
+        },
         {
             "type": "java",
             "name": "RascalCheck",

src/org/rascalmpl/compiler/BreakTest.java

@@ -0,0 +1,235 @@
+package org.rascalmpl.compiler;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.rascalmpl.debug.IRascalMonitor;
+import org.rascalmpl.interpreter.Evaluator;
+import org.rascalmpl.interpreter.result.AbstractFunction;
+import org.rascalmpl.shell.RascalShell;
+import org.rascalmpl.shell.ShellEvaluatorFactory;
+import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner;
+import org.rascalmpl.uri.URIResolverRegistry;
+
+import io.usethesource.vallang.IBool;
+import io.usethesource.vallang.ISourceLocation;
+
+public class BreakTest {
+
+    // private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::ChangeScenarioTests";
+    // private static final String BREAKING_TEST = "fixedErrorsDisappear2";
+    private static final String BREAKING_MODULE = "lang::rascalcore::check::tests::FunctionTCTests";
+    private static final String BREAKING_TEST = "BoundOK1";
+
+    // notCompatibleAfterChangingFunctionArgument
+    // notCompatibleAfterChangingFunctionArgument
+
+    private static final int PARALLEL_RUNS = 1; // set to 1 to avoid any multi-threading interactions, but it might take 20 rounds or something
+    private static final int TRIES = 1000 / PARALLEL_RUNS;
+
+    public static void main(String[] args) throws IOException, InterruptedException {
+        RascalShell.setupJavaProcessForREPL();
+
+        var term = RascalShell.connectToTerminal();
+        var monitor = IRascalMonitor.buildConsoleMonitor(term);
+        var error = monitor instanceof PrintWriter ? (PrintWriter) monitor : new PrintWriter(System.err, false);
+        AtomicBoolean failed = new AtomicBoolean(false);
+        try {
+            AtomicInteger done = new AtomicInteger(0);
+            for (int t = 0; t < PARALLEL_RUNS; t++) {
+                var name = "Thread " + (t + 1);
+                var tr = new Thread(() -> {
+                    try {
+                        if (crashTestFreshEval(monitor, error, name, failed)) {
+                            failed.set(true);
+                            error.flush();
+                            System.err.println("We got a failure, exiting now!");
+                            Thread.sleep(1000);
+                        }
+                    }
+                    catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    } finally {
+                        done.incrementAndGet();
+                    }
+                });
+                tr.start();
+            }
+            while (done.get() < PARALLEL_RUNS && !failed.get()) {
+                Thread.sleep(100);
+            }
+        } finally {
+            error.flush();
+            error.close();
+        }
+        if (failed.get()) {
+            System.out.flush();
+            System.err.flush();
+            System.exit(1);
+        }
+    }
+
+    static boolean crashTest(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) {
+        var output = new StringWriter();
+        var iFailed = new AtomicBoolean(false);
+        try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) {
+            var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class);
+            var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$");
+            evaluator.getConfiguration().setErrors(true);
+            // make sure we're writing to the outputs
+            evaluator.overwritePrintWriter(out, err);
+
+            evaluator.doImport(monitor, BREAKING_MODULE);
+            try {
+                monitor.job(name, TRIES, (jobname, step) -> {
+                    String currentTest = "";
+                    var tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests();
+                    try {
+                        tests = tests.stream()
+                            .filter(t -> !t.hasTag("ignore"))
+                            .collect(Collectors.toList());
+
+                        monitor.jobTodo(jobname, tests.size() * TRIES);
+
+                        for (int i = 0; i < TRIES; i++) {
+                            if (failed.get()) {
+                                return false;
+                            }
+
+                            Collections.shuffle(tests);
+
+                            monitor.jobStep(jobname, "Running: try " + (i + 1));
+                            for (var t: tests) {
+                                currentTest = t.getName();
+                                monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest);
+                                var result = t.call();
+                                if (!((IBool)result).getValue()) {
+                                    throw new RuntimeException("Test " + currentTest +" returned false");
+                                }
+                            }
+                        }
+
+                    } catch (Throwable e ) {
+                        err.println("❌ test fail :" + currentTest);
+                        err.println(e);
+                        e.printStackTrace(err);
+                        iFailed.set(true);
+                    }
+                    return null;
+                });
+            } finally {
+                // clean up memory
+                var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs");
+                if (memoryModule != null ) {
+                var testRoot = memoryModule.getFrameVariable("testRoot");
+                try {
+                    URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true);
+                }
+                catch (Throwable e) {
+                    err.println("Failure to cleanup the cache");
+                }
+            }
+            }
+
+        }
+
+        if (iFailed.get()) {
+            errorPrinter.println("❌❌❌ Test run failed: " + name);
+            errorPrinter.println("Job output:");
+            errorPrinter.println(output.toString());
+            failed.set(true);
+            return true;
+        }
+        return false;
+    }
+
+    static boolean crashTestFreshEval(IRascalMonitor monitor, PrintWriter errorPrinter, String name, AtomicBoolean failed) {
+        var output = new StringWriter();
+        var iFailed = new AtomicBoolean(false);
+        try (var err = new PrintWriter(output, true); var out= new PrintWriter(output, false)) {
+            var projectRoot = RascalJUnitTestRunner.inferProjectRootFromClass(BreakTest.class);
+            monitor.job(name, TRIES * 2, (jobname, step) -> {
+                String currentTest = "";
+                var tests = Collections.<AbstractFunction>emptyList();
+                    try {
+                        for (int i = 0; i < TRIES; i++) {
+                            if (failed.get()) {
+                                return false;
+                            }
+                            monitor.jobStep(jobname, "Try " + (i + 1));
+                            var evaluator = ShellEvaluatorFactory.getDefaultEvaluatorForLocation(projectRoot, Reader.nullReader(), err, out, monitor, "$test-"+name+"$");
+                            try {
+
+                                evaluator.getConfiguration().setErrors(true);
+                                // make sure we're writing to the outputs
+                                evaluator.overwritePrintWriter(out, err);
+
+                                evaluator.doImport(monitor, BREAKING_MODULE);
+
+                                tests = evaluator.getHeap().getModule(BREAKING_MODULE).getTests()
+                                    .stream()
+                                    .filter(t -> !t.hasTag("ignore"))
+                                    .collect(Collectors.toList());
+
+                                if (i == 0) {
+                                    monitor.jobTodo(jobname, tests.size() * TRIES);
+                                }
+
+                                Collections.shuffle(tests);
+
+                                monitor.jobStep(jobname, "Running: try " + (i + 1));
+                                for (var t: tests) {
+                                    currentTest = t.getName();
+                                    monitor.jobStep(jobname, "Running: try " + (i + 1) + " => " + currentTest);
+                                    var result = t.call();
+                                    if (!((IBool)result).getValue()) {
+                                        throw new RuntimeException("Test " + currentTest +" returned false");
+                                    }
+                                }
+                            } finally {
+                                // clean up memory
+                                var memoryModule = evaluator.getHeap().getModule("lang::rascalcore::check::TestConfigs");
+                                if (memoryModule != null) {
+                                    var testRoot = memoryModule.getFrameVariable("testRoot");
+                                    try {
+                                        URIResolverRegistry.getInstance().remove((ISourceLocation)testRoot.getValue(), true);
+                                    }
+                                    catch (Throwable e) {
+                                        err.println("Failure to cleanup the cache");
+                                    }
+                                }
+                            }
+                        }
+
+                } catch (Throwable e ) {
+                    iFailed.set(true);
+                    err.println("tests: " + Arrays.deepToString(tests.stream().map(AbstractFunction::getName).toArray()));
+                    err.println("❌ test fail :" + currentTest);
+                    err.println(e);
+                    e.printStackTrace(err);
+                }
+                return null;
+            });
+        }
+
+        if (iFailed.get()) {
+            errorPrinter.println("❌❌❌ Test run failed: " + name);
+            errorPrinter.println("Job output:");
+            errorPrinter.println(output.toString());
+            errorPrinter.flush();
+            failed.set(true);
+            return true;
+        }
+        return false;
+    }
+}

src/org/rascalmpl/compiler/lang/rascalcore/check/TestConfigs.rsc

@@ -34,6 +34,10 @@ import lang::rascalcore::check::RascalConfig;
 import lang::rascalcore::check::ModuleLocations;
 // import lang::rascalcore::CompilerPathConfig;
 
+import util::UUID;
+
+
+
 // Duplicate in lang::rascalcore::compile::util::Names, factor out
 data PathConfig(
     loc generatedSources=|unknown:///|,
@@ -64,6 +68,9 @@ loc TMP_COMPILED_RASCAL = |tmp:///compiled-rascal/|;
 
 // ---- PathConfigs for testing purposes --------------------------------------
 
+public loc testRoot = uuid()[scheme="memory"];
+public loc testModulesRoot = testRoot + "src";
+
 private int npc = 0;
 @synopsis{PathConfig for testing generated modules in |memory://test-modules/| in memory file system, not depending on any outside libraries.}
 @description{
@@ -74,10 +81,10 @@ public PathConfig getDefaultTestingPathConfig() {
     npc += 1;
     snpc = "<npc>";
     return pathConfig(
-        srcs = [ |memory:///test-modules/|, |std:///|  ],
-        bin = |memory:///test-modules/rascal-tests-bin-<snpc>|,
-        generatedSources = |memory:///test-modules/generated-test-sources-<snpc>|,
-        generatedResources = |memory:///test-modules/generated-test-resources-<snpc>|,
+        srcs = [ testModulesRoot, |std:///|  ],
+        bin = testRoot + "rascal-tests-bin-<snpc>",
+        generatedSources = testRoot + "generated-test-sources-<snpc>",
+        generatedResources = testRoot + "generated-test-resources-<snpc>",
         libs = [ ]
     );
 }
@@ -92,10 +99,10 @@ public PathConfig getReleasedStandardLibraryTestingPathConfig() {
     npc += 1;
     snpc = "<npc>";
     return pathConfig(
-        srcs = [ |memory:///test-modules/| ],
-        bin = |memory:///test-modules/rascal-tests-bin-<snpc>|,
-        generatedSources = |memory:///test-modules/generated-test-sources-<snpc>|,
-        generatedResources = |memory:///test-modules/generated-test-resources-<snpc>|,
+        srcs = [ testModulesRoot ],
+        bin = testRoot + "rascal-tests-bin-<snpc>",
+        generatedSources = testRoot + "generated-test-sources-<snpc>",
+        generatedResources = testRoot + "generated-test-resources-<snpc>",
         libs = [ |lib://rascal| ]
     );
 }
@@ -108,7 +115,7 @@ public PathConfig getRascalProjectTestingPathConfig() {
     snpc = "<npc>";
     return pathConfig(
         srcs = [|project://rascal/src/org/rascalmpl/library|],
-        bin = |memory:///test-modules/rascal-lib-bin-<snpc>|,
+        bin = testRoot + "rascal-lib-bin-<snpc>",
         libs = []
     );
 }

src/org/rascalmpl/compiler/lang/rascalcore/check/tests/BinaryDependencyTests.rsc

@@ -58,10 +58,10 @@ data PathConfig(loc generatedResources=|unknown:///|, loc generatedSources=|unkn
 data Project
     = project(str name, map[str moduleName, str moduleText] modules, PathConfig pcfg);
 
-void clearMemory() { remove(|memory:///|, recursive = true); }
+void clearMemory() { remove(testRoot, recursive = true); }
 
 loc projectDir(str pname)
-    = |memory://<pname>/|;
+    = testRoot + pname;
 
 loc src(str pname)
     = projectDir(pname) + "src/";

src/org/rascalmpl/compiler/lang/rascalcore/check/tests/ChangeScenarioTests.rsc

@@ -882,4 +882,4 @@ void allBenchmarks(){
     mediumBenchmarkRechecking();
     //largeBenchmarkRechecking();
     println("Total time: <(cpuTime() - beginTime)/1000000> ms");
-}
+}

src/org/rascalmpl/compiler/lang/rascalcore/check/tests/StaticTestingUtils.rsc

@@ -75,22 +75,22 @@ tuple[str,str] extractModuleNameAndBody(str moduleText){
 
 loc composeModule(str stmts){
     return writeModule(
-        "module TestModule
+        "module TestModule<md5Hash(stmts)[..5]>
         'value main(){
         '    <stmts>\n
         '    return true;
         '}");
 }
 
 void clearMemory() {
-	remove(|memory:///test-modules/| recursive = true);
+	remove(testRoot, recursive = true);
 }
 str cleanName(str name)
 	= name[0] == "\\" ? name[1..] : name;
 
 loc writeModule(str moduleText){
 	<mname, mbody> = extractModuleNameAndBody(moduleText);
-    mloc = |memory:///test-modules/<cleanName(mname)>.rsc|;
+    mloc = testModulesRoot +  "<cleanName(mname)>.rsc";
     writeFile(mloc, moduleText);
     return mloc;
 }
@@ -101,23 +101,22 @@ list[loc] writeModules(str modules...)
 void removeModule(str mname){
 	pcfg = getDefaultTestingPathConfig();
 	name = cleanName(mname);
-	remove(|memory:///test-modules/<name>.rsc|);
+	remove(testModulesRoot + "<name>.rsc");
 	remove(pcfg.generatedResources + "<name>.tpl");
 }
 
 void printModules(){
 	println("\<\<\<\<");
 	for(f <- find(testRoot, "rsc")){
 		println("<f> <lastModified(f)>:
-				'<readFile(f)>");
+		        '<readFile(f)>");
 	}
 	for(f <- find(testRoot, "tpl")){
 		println("<f>: <lastModified(f)>");
 	}
 	println("\>\>\>\>");
 }
 
-
 set[Message] getErrorMessages(ModuleStatus r)
     =  { m | m <- getAllMessages(r), m is error };
 
@@ -573,4 +572,4 @@ bool expectReChecksWithErrors(loc mloc, list[str] moduleNames, PathConfig pathCo
 	msgs = [ "Checked <nm>" | nm <- moduleNames ];
 	mlocs = mloc + [ getModuleLocation(mn, pathConfig) | mn <- moduleNames ];
 	return checkModuleAndFilter(mlocs, msgs, matchAll=true, errorsAllowed=true, pathConfig=pathConfig);
-}
+}