Change regex generator in fuzzing (#2318)

* Change regex generator library

* Introduce an option to choose a ratio between generation and mutation

Author: Markoutte

utbot-fuzzing/build.gradle.kts

@@ -3,5 +3,5 @@ val rgxgenVersion: String by rootProject
 
 dependencies {
     implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
-    implementation(group = "com.github.curious-odd-man", name = "rgxgen", version = rgxgenVersion)
+    implementation(group = "org.cornutum.regexp", name = "regexp-gen", version = "2.0.1")
 }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt

@@ -324,18 +324,21 @@ private suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R,
 
     while (!fuzzing.isCancelled(description, statistic)) {
         beforeIteration(description, statistic)
-        var values = if (statistic.isNotEmpty() && random.flipCoin(configuration.probSeedRetrievingInsteadGenerating)) {
-            statistic.getRandomSeed(random, configuration)
+        val values = if (statistic.isNotEmpty() && random.flipCoin(configuration.probSeedRetrievingInsteadGenerating)) {
+            statistic.getRandomSeed(random, configuration).let {
+                mutationFactory.mutate(it, random, configuration)
+            }
         } else {
             val actualParameters = description.parameters
             // fuzz one value, seems to be bad, when have only a few and simple values
-            fuzzOne(actualParameters)
+            fuzzOne(actualParameters).let {
+                if (random.flipCoin(configuration.probMutationRate)) {
+                    mutationFactory.mutate(it, random, configuration)
+                } else {
+                    it
+                }
+            }
         }
-        values = mutationFactory.mutate(
-            values,
-            random,
-            configuration
-        )
         afterIteration(description, statistic)
 
         yield()

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt

@@ -12,6 +12,11 @@ data class Configuration(
      */
     var probSeedRetrievingInsteadGenerating: Int = 70,
 
+    /**
+     * Choose between generation and mutation.
+     */
+    var probMutationRate: Int = 99,
+
     /**
      * Fuzzer creates a tree of object for generating values. At some point this recursion should be stopped.
      *

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Mutations.kt

@@ -2,10 +2,7 @@
 
 package org.utbot.fuzzing
 
-import org.utbot.fuzzing.seeds.BitVectorValue
-import org.utbot.fuzzing.seeds.IEEE754Value
-import org.utbot.fuzzing.seeds.KnownValue
-import org.utbot.fuzzing.seeds.StringValue
+import org.utbot.fuzzing.seeds.*
 import org.utbot.fuzzing.utils.chooseOne
 import org.utbot.fuzzing.utils.flipCoin
 import kotlin.random.Random
@@ -190,7 +187,7 @@ sealed interface StringMutations : Mutation<StringValue> {
                 }
                 append(value.substring(position, value.length))
             }
-            return StringValue(newString, lastMutation = this)
+            return StringValue(newString, lastMutation = this, mutatedFrom = source)
         }
     }
 
@@ -207,6 +204,15 @@ sealed interface StringMutations : Mutation<StringValue> {
             return StringValue(newString, this)
         }
     }
+
+    object ShuffleCharacters : StringMutations {
+        override fun mutate(source: StringValue, random: Random, configuration: Configuration): StringValue {
+            return StringValue(
+                value = String(source.value.toCharArray().apply { shuffle(random) }),
+                lastMutation = this
+            )
+        }
+    }
 }
 
 fun interface NodeMutation<TYPE, RESULT> : Mutation<Node<TYPE, RESULT>>

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/RegexValue.kt

@@ -1,17 +1,19 @@
 package org.utbot.fuzzing.seeds
 
-import com.github.curiousoddman.rgxgen.RgxGen
-import com.github.curiousoddman.rgxgen.config.RgxGenOption
-import com.github.curiousoddman.rgxgen.config.RgxGenProperties
+import org.cornutum.regexpgen.js.Provider
+import org.cornutum.regexpgen.random.RandomBoundsGen
 import org.utbot.fuzzing.Mutation
 import kotlin.random.Random
 import kotlin.random.asJavaRandom
 
-class RegexValue(val pattern: String, val random: Random) : StringValue(
+class RegexValue(
+    val pattern: String,
+    val random: Random,
+    val maxLength: Int = listOf(16, 256, 2048).random(random)
+) : StringValue(
     valueProvider = {
-        RgxGen(pattern).apply {
-            setProperties(rgxGenProperties)
-        }.generate(random.asJavaRandom())
+        val matchingExact = Provider.forEcmaScript().matchingExact(pattern)
+        matchingExact.generate(RandomBoundsGen(random.asJavaRandom()), 1, maxLength)
     }
 ) {
 
@@ -22,6 +24,12 @@ class RegexValue(val pattern: String, val random: Random) : StringValue(
     }
 }
 
-private val rgxGenProperties = RgxGenProperties().apply {
-    setProperty(RgxGenOption.INFINITE_PATTERN_REPETITION.key, "100")
+fun String.isSupportedPattern(): Boolean {
+    if (isEmpty()) return false
+    return try {
+        Provider.forEcmaScript().matchingExact(this)
+        true
+    } catch (_: Throwable) {
+        false
+    }
 }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/StringValue.kt

@@ -5,17 +5,23 @@ import org.utbot.fuzzing.StringMutations
 
 open class StringValue(
     val valueProvider: () -> String,
-    override val lastMutation: Mutation<out StringValue>? = null
+    override val lastMutation: Mutation<out StringValue>? = null,
+    override val mutatedFrom: StringValue? = null,
 ) : KnownValue<StringValue> {
 
-    constructor(value: String, lastMutation: Mutation<out StringValue>? = null) : this(valueProvider = { value }, lastMutation)
+    constructor(
+        value: String,
+        lastMutation: Mutation<out StringValue>? = null,
+        mutatedFrom: StringValue? = null
+    ) : this(valueProvider = { value }, lastMutation, mutatedFrom)
 
     val value by lazy { valueProvider() }
 
     override fun mutations(): List<Mutation<out StringValue>> {
         return listOf(
             StringMutations.AddCharacter,
             StringMutations.RemoveCharacter,
+            StringMutations.ShuffleCharacters,
         )
     }
 }

utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/RandomExtensions.kt

@@ -27,6 +27,8 @@ fun Random.chooseOne(frequencies: DoubleArray): Int {
  * If a random value is less than [probability] returns true.
  */
 fun Random.flipCoin(probability: Int): Boolean {
+    if (probability == 0) return false
+    if (probability == 100) return true
     check(probability in 0 .. 100) { "probability must in range [0, 100] but $probability is provided" }
     return nextInt(1, 101) <= probability
 }

utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt

@@ -13,7 +13,7 @@ import kotlin.reflect.KClass
 class FuzzerSmokeTest {
 
     @Test
-    fun `fuzzing doesn't run with empty parameters`() {
+    fun `fuzzing runs with empty parameters`() {
         runBlocking {
             var count = 0
             runFuzzing<Unit, Unit, Description<Unit>, BaseFeedback<Unit, Unit, Unit>>(
@@ -23,7 +23,7 @@ class FuzzerSmokeTest {
                 count += 1
                 BaseFeedback(Unit, Control.STOP)
             }
-            Assertions.assertEquals(0, count)
+            Assertions.assertEquals(1, count)
         }
     }
 

utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Primitives.kt

@@ -65,12 +65,12 @@ abstract class PrimitiveValueProvider(
                     else -> toString()
                 }
             }
-            is StringValue -> {
-                return "'${value.substringToLength(10, "...")}'"
-            }
             is RegexValue -> {
                 return "'${value.substringToLength(10, "...")}' from $pattern"
             }
+            is StringValue -> {
+                return "'${value.substringToLength(10, "...")}'"
+            }
             else -> return toString()
         }
     }
@@ -221,14 +221,8 @@ object StringValueProvider : PrimitiveValueProvider(stringClassId, java.lang.Cha
             .filter { it.fuzzedContext.isPatterMatchingContext()  }
             .map { it.value as String }
             .distinct()
-            .filter { it.isNotBlank() }
-            .filter {
-                try {
-                    Pattern.compile(it); true
-                } catch (_: PatternSyntaxException) {
-                    false
-                }
-            }.forEach {
+            .filter(String::isSupportedPattern)
+            .forEach {
                 yieldKnown(RegexValue(it, Random(0)), StringValue::value)
             }
     }

utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/utils/StringUtils.kt

@@ -1,7 +1,6 @@
 package org.utbot.python.fuzzing.provider.utils
 
-import java.util.regex.Pattern
-import java.util.regex.PatternSyntaxException
+import org.utbot.fuzzing.seeds.isSupportedPattern
 
 fun String.transformQuotationMarks(): String {
 
@@ -39,12 +38,6 @@ fun String.isRawString(): Boolean {
 fun String.isPattern(): Boolean {
     return if (this.isRawString()) {
         val stringContent = this.transformRawString()
-        if (stringContent.isNotBlank()) {
-            try {
-                Pattern.compile(stringContent); true
-            } catch (_: PatternSyntaxException) {
-                false
-            }
-        } else false
+        return stringContent.isSupportedPattern()
     } else false
 }