Exclude third-party libs from coverage statistics #2218 (#2282)

Author: sofurihafe

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -632,7 +632,7 @@ enum class TestSelectionStrategyType {
     /**
      * Adds new test only if it increases coverage
      */
-    COVERAGE_STRATEGY
+    COVERAGE_STRATEGY,
 }
 
 /**

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -140,7 +140,17 @@ abstract class UtExecution(
     var summary: List<DocStatement>? = null,
     var testMethodName: String? = null,
     var displayName: String? = null
-) : UtResult()
+) : UtResult() {
+    abstract fun copy(
+        stateBefore: EnvironmentModels = this.stateBefore,
+        stateAfter: EnvironmentModels = this.stateAfter,
+        result: UtExecutionResult = this.result,
+        coverage: Coverage? = this.coverage,
+        summary: List<DocStatement>? = this.summary,
+        testMethodName: String? = this.testMethodName,
+        displayName: String? = this.displayName,
+    ): UtExecution
+}
 
 /**
  * Symbolic execution.
@@ -176,6 +186,27 @@ class UtSymbolicExecution(
 
     var containsMocking: Boolean = false
 
+    override fun copy(
+        stateBefore: EnvironmentModels,
+        stateAfter: EnvironmentModels,
+        result: UtExecutionResult,
+        coverage: Coverage?,
+        summary: List<DocStatement>?,
+        testMethodName: String?,
+        displayName: String?
+    ): UtExecution = UtSymbolicExecution(
+        stateBefore = stateBefore,
+        stateAfter = stateAfter,
+        result = result,
+        instrumentation = instrumentation,
+        path = path,
+        fullPath = fullPath,
+        coverage = coverage,
+        summary = summary,
+        testMethodName = testMethodName,
+        displayName = displayName
+    )
+
     override fun toString(): String = buildString {
         append("UtSymbolicExecution(")
         appendLine()
@@ -200,25 +231,28 @@ class UtSymbolicExecution(
     }
 
     fun copy(
-        stateAfter: EnvironmentModels,
-        result: UtExecutionResult,
-        coverage: Coverage,
+        stateBefore: EnvironmentModels = this.stateBefore,
+        stateAfter: EnvironmentModels = this.stateAfter,
+        result: UtExecutionResult = this.result,
+        coverage: Coverage? = this.coverage,
+        summary: List<DocStatement>? = this.summary,
+        testMethodName: String? = this.testMethodName,
+        displayName: String? = this.displayName,
         instrumentation: List<UtInstrumentation> = this.instrumentation,
-    ): UtResult {
-        return UtSymbolicExecution(
-            stateBefore,
-            stateAfter,
-            result,
-            instrumentation,
-            path,
-            fullPath,
-            coverage,
-            summary,
-            testMethodName,
-            displayName,
-            symbolicSteps,
-        )
-    }
+        path: MutableList<Step> = this.path,
+        fullPath: List<Step> = this.fullPath
+    ): UtExecution = UtSymbolicExecution(
+        stateBefore = stateBefore,
+        stateAfter = stateAfter,
+        result = result,
+        instrumentation = instrumentation,
+        path = path,
+        fullPath = fullPath,
+        coverage = coverage,
+        summary = summary,
+        testMethodName = testMethodName,
+        displayName = displayName
+    )
 }
 
 /**
@@ -235,12 +269,31 @@ class UtSymbolicExecution(
  */
 class UtFailedExecution(
     stateBefore: EnvironmentModels,
-    result: UtExecutionFailure,
+    result: UtExecutionResult,
     coverage: Coverage? = null,
     summary: List<DocStatement>? = null,
     testMethodName: String? = null,
     displayName: String? = null
-) : UtExecution(stateBefore, MissingState, result, coverage, summary, testMethodName, displayName)
+) : UtExecution(stateBefore, MissingState, result, coverage, summary, testMethodName, displayName) {
+    override fun copy(
+        stateBefore: EnvironmentModels,
+        stateAfter: EnvironmentModels,
+        result: UtExecutionResult,
+        coverage: Coverage?,
+        summary: List<DocStatement>?,
+        testMethodName: String?,
+        displayName: String?
+    ): UtExecution {
+        return UtFailedExecution(
+            stateBefore,
+            result,
+            coverage,
+            summary,
+            testMethodName,
+            displayName
+        )
+    }
+}
 
 open class EnvironmentModels(
     val thisInstance: UtModel?,

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt

@@ -3,7 +3,7 @@ package org.utbot.framework.plugin.api
 /**
  * Represents a covered bytecode instruction.
  *
- * @param className a fqn of the class.
+ * @param className the fqn in internal form, i.e. com/rest/order/services/OrderService$InnerClass.
  * @param methodSignature the signature of the method.
  * @param lineNumber a number of the line in the source file.
  * @param id a unique identifier among all instructions in all classes.

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/util/SpringModelUtils.kt

@@ -12,6 +12,7 @@ import org.utbot.framework.plugin.api.UtSpringContextModel
 object SpringModelUtils {
     val applicationContextClassId = ClassId("org.springframework.context.ApplicationContext")
     val crudRepositoryClassId = ClassId("org.springframework.data.repository.CrudRepository")
+    val entityClassId = ClassId("javax.persistence.Entity")
 
     val getBeanMethodId = MethodId(
         classId = applicationContextClassId,

utbot-framework/src/main/kotlin/org/utbot/framework/minimization/Minimization.kt

@@ -238,8 +238,8 @@ private fun UtModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): I
 
 private fun UtStatementModel.calculateSize(used: MutableSet<UtModel> = mutableSetOf()): Int =
     when (this) {
-        is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used)
-        is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) }
+        is UtDirectSetFieldModel -> 1 + fieldModel.calculateSize(used) + instance.calculateSize(used)
+        is UtStatementCallModel -> 1 + params.sumOf { it.calculateSize(used) } + (instance?.calculateSize(used) ?: 0)
     }
 
 /**

utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt

@@ -14,6 +14,8 @@ import org.utbot.common.measureTime
 import org.utbot.common.runBlockingWithCancellationPredicate
 import org.utbot.common.runIgnoringCancellationException
 import org.utbot.common.trace
+import org.utbot.common.PathUtil.toURL
+import org.utbot.common.tryLoadClass
 import org.utbot.engine.EngineController
 import org.utbot.engine.Mocker
 import org.utbot.engine.UtBotSymbolicEngine
@@ -27,6 +29,7 @@ import org.utbot.framework.UtSettings.utBotGenerationTimeoutInMillis
 import org.utbot.framework.UtSettings.warmupConcreteExecution
 import org.utbot.framework.plugin.api.utils.checkFrameworkDependencies
 import org.utbot.framework.minimization.minimizeTestCase
+import org.utbot.framework.plugin.api.util.SpringModelUtils.entityClassId
 import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.utContext
 import org.utbot.framework.plugin.services.JdkInfo
@@ -41,6 +44,7 @@ import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrument
 import org.utbot.instrumentation.warmup
 import org.utbot.taint.TaintConfigurationProvider
 import java.io.File
+import java.net.URLClassLoader
 import java.nio.file.Path
 import kotlin.coroutines.cancellation.CancellationException
 import kotlin.math.min
@@ -116,11 +120,17 @@ open class TestCaseGenerator(
         }
     }
 
-    fun minimizeExecutions(executions: List<UtExecution>): List<UtExecution> =
-        if (UtSettings.testMinimizationStrategyType == TestSelectionStrategyType.DO_NOT_MINIMIZE_STRATEGY) {
-            executions
-        } else {
-            minimizeTestCase(executions) { it.result::class.java }
+    fun minimizeExecutions(classUnderTestId: ClassId, executions: List<UtExecution>): List<UtExecution> =
+        when (UtSettings.testMinimizationStrategyType) {
+            TestSelectionStrategyType.DO_NOT_MINIMIZE_STRATEGY -> executions
+            TestSelectionStrategyType.COVERAGE_STRATEGY ->
+                minimizeTestCase(
+                    when (applicationContext) {
+                        is SpringApplicationContext -> executions.filterCoveredInstructions(classUnderTestId)
+                        else -> executions
+                    }
+                )
+                { it.result::class.java }
         }
 
     @Throws(CancellationException::class)
@@ -268,7 +278,7 @@ open class TestCaseGenerator(
         return methods.map { method ->
             UtMethodTestSet(
                 method,
-                minimizeExecutions(method2executions.getValue(method)),
+                minimizeExecutions(method.classId, method2executions.getValue(method)),
                 jimpleBody(method),
                 method2errors.getValue(method)
             )
@@ -351,6 +361,80 @@ open class TestCaseGenerator(
         }
     }
 
+    private fun List<UtExecution>.filterCoveredInstructions(classUnderTestId: ClassId): List<UtExecution> {
+        // Do nothing when we were launched not from IDEA or when there are no executions
+        if (buildDirs.isEmpty() || this.isEmpty()) return this
+
+        // List of annotations that we want to find in execution instructions
+        // in order to exclude such instructions for some reason
+        // E.g. we exclude instructions (which classes have @Entity) when it is not a class under test
+        // because we are not interested in coverage which was possibly produced by Spring itself
+        val annotationsToIgnore =
+            listOfNotNull(utContext.classLoader.tryLoadClass(entityClassId.name))
+
+        val buildDirsClassLoader = createBuildDirsClassLoader()
+        val isClassOnUserClasspathCache = mutableMapOf<String, Boolean>()
+
+        // Here we filter out instructions from third-party libraries
+        // Also, we filter out instructions that operate
+        // in classes marked with annotations from [annotationsToIgnore]
+        // and in standard java libs
+        return this.map { execution ->
+            val coverage = execution.coverage ?: return@map execution
+
+            val filteredCoveredInstructions =
+                coverage.coveredInstructions
+                    .filter { instruction ->
+                        val instrClassFqn =
+                            instruction.className
+                                .also {
+                                    val isInstrClassOnClassPath =
+                                        isClassOnUserClasspathCache.getOrPut(it) {
+                                            buildDirsClassLoader.findResource(it.plus(".class")) != null
+                                        }
+                                    if (!isInstrClassOnClassPath) return@filter false
+                                }
+                                .replace('/', '.')
+
+                        // We do not want to filter out instructions that are in class under test
+                        if (instrClassFqn == classUnderTestId.name) return@filter true
+
+                        // We do not want to take instructions in classes
+                        // marked with annotations from [annotationsToIgnore]
+                        return@filter !hasAnnotations(instrClassFqn, annotationsToIgnore)
+                    }
+                    .ifEmpty {
+                        coverage.coveredInstructions
+                            .also {
+                                logger.warn("Execution covered instruction list became empty. Proceeding with not filtered instruction list.")
+                            }
+                    }
+
+            execution.copy(
+                coverage = Coverage(
+                    coveredInstructions = filteredCoveredInstructions,
+                    instructionsCount = coverage.instructionsCount,
+                    missedInstructions = coverage.missedInstructions
+                )
+            )
+        }
+    }
+
+    private fun createBuildDirsClassLoader(): URLClassLoader {
+        val urls = buildDirs.map { it.toURL() }.toTypedArray()
+        return URLClassLoader(urls, null)
+    }
+
+    private fun hasAnnotations(className: String, annotations: List<Class<*>>): Boolean =
+        utContext
+            .classLoader
+            .loadClass(className)
+            .annotations
+            .any { existingAnnotation ->
+                annotations.any { annotation ->
+                    annotation.isInstance(existingAnnotation)
+                }
+            }
 }
 
 

utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt

@@ -106,7 +106,8 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
         Instrumenter.adapter = RdInstrumenter(realProtocol.rdInstrumenterAdapter)
         val applicationContext: ApplicationContext = kryoHelper.readObject(params.applicationContext)
 
-        testGenerator = TestCaseGenerator(buildDirs = params.buildDir.map { Paths.get(it) },
+        testGenerator = TestCaseGenerator(
+            buildDirs = params.buildDir.map { Paths.get(it) },
             classpath = params.classpath,
             dependencyPaths = params.dependencyPaths,
             jdkInfo = JdkInfo(Paths.get(params.jdkInfo.path), params.jdkInfo.version),
@@ -115,7 +116,8 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
                 runBlocking {
                     model.isCancelled.startSuspending(Unit)
                 }
-            })
+            }
+        )
     }
     watchdog.measureTimeForActiveCall(generate, "Generating tests") { params ->
         val methods: List<ExecutableId> = kryoHelper.readObject(params.methods)

utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt

@@ -32,6 +32,29 @@ class UtFuzzedExecution(
     val staticFields: Set<FieldId>
         get() = stateBefore.statics.keys // TODO: should we keep it for the Fuzzed Execution?
 
+    override fun copy(
+        stateBefore: EnvironmentModels,
+        stateAfter: EnvironmentModels,
+        result: UtExecutionResult,
+        coverage: Coverage?,
+        summary: List<DocStatement>?,
+        testMethodName: String?,
+        displayName: String?
+    ): UtExecution {
+        return UtFuzzedExecution(
+            stateBefore,
+            stateAfter,
+            result,
+            coverage,
+            summary,
+            testMethodName,
+            displayName,
+            fuzzingValues = fuzzingValues,
+            fuzzedMethodDescription = fuzzedMethodDescription,
+            instrumentation = instrumentation,
+        )
+    }
+
     override fun toString(): String = buildString {
         append("UtFuzzedExecution(")
         appendLine()
@@ -51,4 +74,28 @@ class UtFuzzedExecution(
         append(result)
         append(")")
     }
+
+    fun copy(
+        stateBefore: EnvironmentModels = this.stateBefore,
+        stateAfter: EnvironmentModels = this.stateAfter,
+        result: UtExecutionResult = this.result,
+        coverage: Coverage? = this.coverage,
+        summary: List<DocStatement>? = this.summary,
+        testMethodName: String? = this.testMethodName,
+        displayName: String? = this.displayName,
+        fuzzingValues: List<FuzzedValue>? = this.fuzzingValues,
+        fuzzedMethodDescription: FuzzedMethodDescription? = this.fuzzedMethodDescription,
+        instrumentation: List<UtInstrumentation> = this.instrumentation,
+    ): UtExecution = UtFuzzedExecution(
+        stateBefore = stateBefore,
+        stateAfter = stateAfter,
+        result = result,
+        coverage = coverage,
+        summary = summary,
+        testMethodName = testMethodName,
+        displayName = displayName,
+        fuzzingValues = fuzzingValues,
+        fuzzedMethodDescription = fuzzedMethodDescription,
+        instrumentation = instrumentation,
+    )
 }