Flaky detect

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -610,6 +610,26 @@ protected void assertTextInTests(String outputFolder, String className, Predicat
 
     }
 
+    /**
+     * assert min count of a certain text in the generated tests
+     * @param outputFolder the folder where the test is
+     * @param className the complete test name
+     * @param condition is the content to check
+     * @param minCount is the minimal count of the certain text existing in the generated tests
+     */
+    protected void assertCountTextInTests(String outputFolder, String className, Predicate<String> condition, int minCount) {
+        String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt";
+        Path test = Paths.get(path);
+        try {
+            boolean ok = Files.lines(test).filter(condition).count() >= minCount;
+            String msg = "Cannot find "+minCount+" lines with requested condition in "+className+" in "+outputFolder;
+            assertTrue(ok, msg);
+        }catch (IOException e){
+            throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage());
+        }
+
+    }
+
     /**
      * This function is used to retrieve the item with the identifier "key" in the Solution "sol"
      * from Statistics. Returns the value of the element if the element exists, otherwise returns null.

core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/flakinessdetect/FlakinessDetectApplication.kt

@@ -0,0 +1,15 @@
+package com.foo.rest.examples.bb.flakinessdetect
+
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+
+@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
+open class FlakinessDetectApplication {
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) {
+            SpringApplication.run(FlakinessDetectApplication::class.java, *args)
+        }
+    }
+}

core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/flakinessdetect/FlakinessDetectRest.kt

@@ -0,0 +1,95 @@
+package com.foo.rest.examples.bb.flakinessdetect
+
+import org.evomaster.e2etests.utils.CoveredTargets
+import org.h2.util.MathUtils.randomInt
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+import org.springframework.web.bind.annotation.RestController
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import kotlin.math.max
+import kotlin.math.min
+
+@RestController
+@RequestMapping(path = ["/api/flakinessdetect"])
+class FlakinessDetectRest {
+
+    companion object{
+        val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww")
+    }
+
+    @GetMapping(path = ["/stringfirst/{n}"])
+    open fun getFirst( @PathVariable("n") n: Int) : ResponseEntity<String> {
+        CoveredTargets.cover("First")
+        return ResponseEntity.ok(getPartialDate(n))
+    }
+
+    @GetMapping(path = ["/next/{n}"])
+    open fun getNext( @PathVariable("n") n: Int) : ResponseEntity<FlakinessDetectData> {
+        CoveredTargets.cover("Next")
+        return ResponseEntity.ok(FlakinessDetectData(getPartialDate(n), randomInt(n)))
+    }
+
+    @GetMapping(path = ["/multiplelines/{num}"])
+    open fun getMultipleLines( @PathVariable("num") num: Int) : ResponseEntity<FlakinessDetectData> {
+
+        val num = min(20, max(2, randomInt(num)))
+
+        val msg = (1..num).joinToString(System.lineSeparator()) { "LINE $it" }
+        CoveredTargets.cover("MultipleLines")
+        return ResponseEntity.ok(FlakinessDetectData(msg, num))
+    }
+
+    @GetMapping("/price/estimate")
+    fun estimatePrice(@RequestParam base: Int): Map<String, Int> {
+        val randomJitter = randomInt(1000)
+
+        val total = base + randomJitter
+
+        CoveredTargets.cover("estimate")
+        return mapOf(
+            "base" to base,
+            "jitter" to randomJitter,
+            "total" to total
+        )
+    }
+
+    @GetMapping("/calculate/timeago")
+    fun getTimeAgo(
+        @RequestParam(defaultValue = "0") day: Long,
+        @RequestParam(defaultValue = "0") hour: Long,
+        @RequestParam(defaultValue = "0") min: Long,
+        @RequestParam(defaultValue = "0") sec: Long
+    ): ResponseEntity<TimeAgoData> {
+
+        CoveredTargets.cover("TimeAgo")
+        val msg = listOf(day to "day", hour to "hour", min to "minute", sec to "second")
+            .filter { it.first > 0 }
+            .joinToString(", ") { (v, u) -> "$v $u${if (v > 1) "s" else ""}" }.takeIf { it.isNotEmpty() }
+            ?.let { "$it ago" }
+            ?: "just now"
+
+        val pastDate = LocalDateTime.now()
+            .minusDays(day).minusHours(hour).minusMinutes(min).minusSeconds(sec)
+
+        return ResponseEntity.ok(TimeAgoData(msg, pastDate.toString()))
+    }
+
+    private fun getPartialDate(n: Int) : String {
+        val now = LocalDateTime.now().format(formatter)
+        val size = max(12, min(now.toString().length, n))
+
+        return now.substring(0, size)
+    }
+}
+
+data class FlakinessDetectData(
+    val first : String, val next : Int)
+
+data class TimeAgoData(
+    val message: String,
+    val calculatedPastTime: String
+)

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/flakinessdetect/FlakinessDetectController.kt

@@ -0,0 +1,7 @@
+package com.foo.rest.examples.bb.flakinessdetect
+
+import com.foo.rest.examples.bb.SpringController
+
+
+
+class FlakinessDetectController : SpringController(FlakinessDetectApplication::class.java)

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt

@@ -0,0 +1,46 @@
+package org.evomaster.e2etests.spring.rest.bb.flakinessdetect
+
+
+import com.foo.rest.examples.bb.flakinessdetect.FlakinessDetectController
+import org.evomaster.core.output.OutputFormat
+import org.evomaster.e2etests.spring.rest.bb.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.EnumSource
+
+class FlakinessDetectBBEMTest : SpringTestBase() {
+
+    companion object {
+        init {
+            shouldApplyInstrumentation = false
+        }
+
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(FlakinessDetectController())
+        }
+    }
+
+
+    @ParameterizedTest
+    @EnumSource
+    fun testBlackBoxOutput(outputFormat: OutputFormat) {
+        executeAndEvaluateBBTest(
+            outputFormat,
+            "flakinessdetect",
+            100,
+            3,
+            listOf("TimeAgo", "estimate","MultipleLines","Next","First")
+        ){ args: MutableList<String> ->
+
+            setOption(args, "handleFlakiness", "true")
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.isNotEmpty())
+
+        }
+    }
+}

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt

@@ -0,0 +1,15 @@
+package com.foo.rest.examples.spring.openapi.v3.flakinessdetect
+
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+
+@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
+open class FlakinessDetectApplication {
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) {
+            SpringApplication.run(FlakinessDetectApplication::class.java, *args)
+        }
+    }
+}

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt

@@ -0,0 +1,91 @@
+package com.foo.rest.examples.spring.openapi.v3.flakinessdetect
+
+import org.h2.util.MathUtils.randomInt
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
+import org.springframework.web.bind.annotation.RestController
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import kotlin.math.max
+import kotlin.math.min
+
+@RestController
+@RequestMapping(path = ["/api/flakinessdetect"])
+class FlakinessDetectRest {
+
+    companion object{
+        val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww")
+    }
+
+    @GetMapping(path = ["/stringfirst/{n}"])
+    open fun getFirst( @PathVariable("n") n: Int) : ResponseEntity<String> {
+
+        return ResponseEntity.ok(getPartialDate(n))
+    }
+
+    @GetMapping(path = ["/next/{n}"])
+    open fun getNext( @PathVariable("n") n: Int) : ResponseEntity<FlakinessDetectData> {
+
+        return ResponseEntity.ok(FlakinessDetectData(getPartialDate(n), randomInt(n)))
+    }
+
+    @GetMapping(path = ["/multiplelines/{num}"])
+    open fun getMultipleLines( @PathVariable("num") num: Int) : ResponseEntity<FlakinessDetectData> {
+
+        val num = min(20, max(2, randomInt(num)))
+
+        val msg = (1..num).joinToString(System.lineSeparator()) { "LINE $it" }
+        return ResponseEntity.ok(FlakinessDetectData(msg, num))
+    }
+
+    @GetMapping("/price/estimate")
+    fun estimatePrice(@RequestParam base: Int): Map<String, Int> {
+        val randomJitter = randomInt(1000)
+
+        val total = base + randomJitter
+
+        return mapOf(
+            "base" to base,
+            "jitter" to randomJitter,
+            "total" to total
+        )
+    }
+
+    @GetMapping("/calculate/timeago")
+    fun getTimeAgo(
+        @RequestParam(defaultValue = "0") day: Long,
+        @RequestParam(defaultValue = "0") hour: Long,
+        @RequestParam(defaultValue = "0") min: Long,
+        @RequestParam(defaultValue = "0") sec: Long
+    ): ResponseEntity<TimeAgoData> {
+
+        val msg = listOf(day to "day", hour to "hour", min to "minute", sec to "second")
+            .filter { it.first > 0 }
+            .joinToString(", ") { (v, u) -> "$v $u${if (v > 1) "s" else ""}" }.takeIf { it.isNotEmpty() }
+            ?.let { "$it ago" }
+            ?: "just now"
+
+        val pastDate = LocalDateTime.now()
+            .minusDays(day).minusHours(hour).minusMinutes(min).minusSeconds(sec)
+
+        return ResponseEntity.ok(TimeAgoData(msg, pastDate.toString()))
+    }
+
+    private fun getPartialDate(n: Int) : String {
+        val now = LocalDateTime.now().format(formatter)
+        val size = max(12, min(now.toString().length, n))
+
+        return now.substring(0, size)
+    }
+}
+
+data class FlakinessDetectData(
+    val first : String, val next : Int)
+
+data class TimeAgoData(
+    val message: String,
+    val calculatedPastTime: String
+)

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt

@@ -0,0 +1,5 @@
+package com.foo.rest.examples.spring.openapi.v3.flakinessdetect
+
+import com.foo.rest.examples.spring.openapi.v3.SpringController
+
+class FlakinessDetectController : SpringController(FlakinessDetectApplication::class.java)

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt

@@ -0,0 +1,54 @@
+package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect
+
+import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController
+import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import java.util.function.Predicate
+
+class FlakinessDetectBlackboxEMTest : SpringTestBase() {
+
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(FlakinessDetectController())
+        }
+    }
+
+
+    @Test
+    fun testRunEM() {
+        defaultSeed = 123
+
+        val outputFolder = "FlakinessDetectBlackboxEM"
+        val outputClass = "org.foo.FlakinessDetectBlackboxEM"
+        val flakyMark = "Flaky"
+
+        runTestHandlingFlakyAndCompilation(
+            outputFolder,
+            outputClass,
+            100
+        ) { args: MutableList<String> ->
+
+
+
+            setOption(args, "handleFlakiness", "true")
+
+            // we may still need to specify info in non bb-e2etest
+            setOption(args, "blackBox", "true")
+            setOption(args, "bbTargetUrl", baseUrlOfSut)
+            setOption(args, "bbSwaggerUrl", "$baseUrlOfSut/v3/api-docs")
+
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.isNotEmpty())
+            assertTextInTests(outputFolder,outputClass,flakyMark)
+            assertCountTextInTests(outputFolder,outputClass,
+                Predicate { it: String? -> it != null && it.contains(flakyMark) }, 3)
+
+        }
+    }
+}