Regex support for missing predefined character classes and negated character classes

core-tests/e2e-tests/spring/spring-rest-mongo/src/test/java/org/evomaster/e2etests/spring/rest/mongo/findoneby/MongoFindOneByEMTest.java

@@ -34,7 +34,7 @@ public void testFindOneOnGivenEndpoint(String endpoint) throws Throwable {
         runTestHandlingFlaky(
                 "MongoFindOneByEM_" + id,
                 "org.foo.spring.rest.mongo.MongoFindOneByEM"+id,
-                1000,
+                2000,
                 true,
                 (args) -> {
                     setOption(args, "taintForceSelectionOfGenesWithSpecialization", "true");

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/static/openapi-dto-reflective-assert.yaml

@@ -97,6 +97,7 @@ paths:
                 - aDouble
                 - aFloat
                 - aBoolean
+                - aNullableString
       responses:
         '200':
           description: OK

core/src/main/antlr4/org/evomaster/core/parser/RegexJava.g4

@@ -125,9 +125,29 @@ fragment CharacterEscape
  | HexEscapeSequence
  | UnicodeEscapeSequence
  | OctalEscapeSequence
+ | 'p' BRACE_open PosixCharacterClassLabel BRACE_close // this is only implemented in Java at the moment as on JS this
+                                                       // is allowed only while certain flags are enabled
  //| IdentityEscape
  ;
 
+// basic US-ASCII only predefined POSIX character classes
+// https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#:~:text=character%3A%20%5B%5E%5Cw%5D-,POSIX,-character%20classes%20(US
+fragment PosixCharacterClassLabel
+ : 'Lower'
+ | 'Upper'
+ | 'ASCII'
+ | 'Alpha'
+ | 'Digit'
+ | 'Alnum'
+ | 'Punct'
+ | 'Graph'
+ | 'Print'
+ | 'Blank'
+ | 'Cntrl'
+ | 'XDigit'
+ | 'Space'
+ ;
+
 fragment ControlEscape
  //one of f n r t v
  : [aefnrt]
@@ -233,8 +253,10 @@ AtomEscape
  ;
 
 fragment CharacterClassEscape
- //one of d D s S w W
- : [dDsSwW]
+ //one of d D s S w W v V h H
+ // v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively
+ // see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information
+ : [dDsSwWvVhH]
  ;
 
 

core/src/main/kotlin/org/evomaster/core/parser/GenePostgresSimilarToVisitor.kt

@@ -2,6 +2,7 @@ package org.evomaster.core.parser
 
 import org.evomaster.core.search.gene.Gene
 import org.evomaster.core.search.gene.regex.*
+import org.evomaster.core.utils.CharacterRange
 
 /**
  * Created by arcuri82 on 12-Jun-19.
@@ -181,7 +182,7 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
 
         val negated = ctx.CARET() != null
 
-        val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+        val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
 
         val gene = CharacterRangeRxGene(negated, ranges)
 
@@ -191,10 +192,10 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
     override fun visitClassRanges(ctx: PostgresSimilarToParser.ClassRangesContext): VisitResult {
 
         val res = VisitResult()
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.nonemptyClassRanges() != null){
-            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -205,7 +206,7 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
 
     override fun visitNonemptyClassRanges(ctx: PostgresSimilarToParser.NonemptyClassRangesContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         val startText = ctx.classAtom()[0].text
         assert(startText.length == 1) // single chars
@@ -218,15 +219,15 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
             start
         }
 
-        list.add(Pair(start, end))
+        list.add(CharacterRange(start, end))
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -239,27 +240,27 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
 
     override fun visitNonemptyClassRangesNoDash(ctx: PostgresSimilarToParser.NonemptyClassRangesNoDashContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.MINUS() != null){
 
             val start = ctx.classAtomNoDash().text[0]
             val end = ctx.classAtom().text[0]
-            list.add(Pair(start, end))
+            list.add(CharacterRange(start, end))
 
         } else {
 
             val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
-            list.add(Pair(char, char))
+            list.add(CharacterRange(char, char))
         }
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexEcma262Visitor.kt

@@ -1,6 +1,7 @@
 package org.evomaster.core.parser
 
 import org.evomaster.core.search.gene.regex.*
+import org.evomaster.core.utils.CharacterRange
 
 private const val EOF_TOKEN = "<EOF>"
 /**
@@ -238,7 +239,7 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
 
         val negated = ctx.CARET() != null
 
-        val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+        val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
 
         val gene = CharacterRangeRxGene(negated, ranges)
 
@@ -248,10 +249,10 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
     override fun visitClassRanges(ctx: RegexEcma262Parser.ClassRangesContext): VisitResult {
 
         val res = VisitResult()
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.nonemptyClassRanges() != null){
-            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -262,7 +263,7 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
 
     override fun visitNonemptyClassRanges(ctx: RegexEcma262Parser.NonemptyClassRangesContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         val startText = ctx.classAtom()[0].text
         assert(startText.length == 1) // single chars
@@ -275,15 +276,15 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
             start
         }
 
-        list.add(Pair(start, end))
+        list.add(CharacterRange(start, end))
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -296,27 +297,27 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
 
     override fun visitNonemptyClassRangesNoDash(ctx: RegexEcma262Parser.NonemptyClassRangesNoDashContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.MINUS() != null){
 
             val start = ctx.classAtomNoDash().text[0]
             val end = ctx.classAtom().text[0]
-            list.add(Pair(start, end))
+            list.add(CharacterRange(start, end))
 
         } else {
 
             val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
-            list.add(Pair(char, char))
+            list.add(CharacterRange(char, char))
         }
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -1,6 +1,7 @@
 package org.evomaster.core.parser
 
 import org.evomaster.core.search.gene.regex.*
+import org.evomaster.core.utils.CharacterRange
 
 private const val EOF_TOKEN = "<EOF>"
 /**
@@ -232,7 +233,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
                         )
                     )
                 }
-                else -> return VisitResult(CharacterClassEscapeRxGene(txt[1].toString()))
+                else -> return VisitResult(CharacterClassEscapeRxGene(txt.substring(1)))
             }
         }
 
@@ -281,7 +282,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
 
         val negated = ctx.CARET() != null
 
-        val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+        val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
 
         val gene = CharacterRangeRxGene(negated, ranges)
 
@@ -291,10 +292,10 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
     override fun visitClassRanges(ctx: RegexJavaParser.ClassRangesContext): VisitResult {
 
         val res = VisitResult()
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.nonemptyClassRanges() != null){
-            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -305,7 +306,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
 
     override fun visitNonemptyClassRanges(ctx: RegexJavaParser.NonemptyClassRangesContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         val startText = ctx.classAtom()[0].text
         assert(startText.length == 1 || startText.length==2) // single chars or \+ and \. escaped chars
@@ -330,15 +331,15 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
             end = start
         }
 
-        list.add(Pair(start, end))
+        list.add(CharacterRange(start, end))
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
@@ -351,27 +352,27 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
 
     override fun visitNonemptyClassRangesNoDash(ctx: RegexJavaParser.NonemptyClassRangesNoDashContext): VisitResult {
 
-        val list = mutableListOf<Pair<Char,Char>>()
+        val list = mutableListOf<CharacterRange>()
 
         if(ctx.MINUS() != null){
 
             val start = ctx.classAtomNoDash().text[0]
             val end = ctx.classAtom().text[0]
-            list.add(Pair(start, end))
+            list.add(CharacterRange(start, end))
 
         } else {
 
             val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
-            list.add(Pair(char, char))
+            list.add(CharacterRange(char, char))
         }
 
         if(ctx.nonemptyClassRangesNoDash() != null){
-            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }
 
         if(ctx.classRanges() != null){
-            val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
+            val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
             list.addAll(ranges)
         }