Add concrete execution specification

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/UtExecutionResult.kt

@@ -21,6 +21,10 @@ sealed class UtExecutionFailure : UtExecutionResult() {
         get() = exception
 }
 
+data class UtIgnoreFailure(
+    override val exception: Throwable,
+) : UtExecutionFailure()
+
 data class UtOverflowFailure(
     override val exception: Throwable,
 ) : UtExecutionFailure()

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -42,13 +42,15 @@ import org.utbot.fuzzer.*
 import org.utbot.fuzzing.*
 import org.utbot.fuzzing.utils.Trie
 import org.utbot.instrumentation.ConcreteExecutor
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionData
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionData
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionResult
 import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
 import soot.jimple.Stmt
 import soot.tagkit.ParamNamesTag
 import java.lang.reflect.Method
 import kotlin.system.measureTimeMillis
+import org.utbot.instrumentation.instrumentation.execution.data.FuzzingSpecification
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionSpecification
 
 val logger = KotlinLogging.logger {}
 val pathLogger = KotlinLogging.logger(logger.name + ".path")
@@ -341,6 +343,7 @@ class UtBotSymbolicEngine(
             // Currently, fuzzer doesn't work with static methods with empty parameters
             return@flow
         }
+        val concreteExecutorSpecification = FuzzingSpecification()
         val errorStackTraceTracker = Trie(StackTraceElement::toString)
         var attempts = 0
         val attemptsLimit = UtSettings.fuzzingMaxAttempts
@@ -362,7 +365,12 @@ class UtBotSymbolicEngine(
             val initialEnvironmentModels = EnvironmentModels(thisInstance?.model, values.map { it.model }, mapOf())
 
             val concreteExecutionResult: UtConcreteExecutionResult? = try {
-                concreteExecutor.executeConcretely(methodUnderTest, initialEnvironmentModels, listOf())
+                concreteExecutor.executeConcretely(
+                    methodUnderTest,
+                    initialEnvironmentModels,
+                    listOf(),
+                    concreteExecutorSpecification,
+                )
             } catch (e: CancellationException) {
                 logger.debug { "Cancelled by timeout" }; null
             } catch (e: ConcreteExecutionFailureException) {
@@ -374,6 +382,11 @@ class UtBotSymbolicEngine(
             // in case an exception occurred from the concrete execution
             concreteExecutionResult ?: return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
 
+            if (concreteExecutionResult.isExpectedFailure()) {
+                logger.debug { "Expected failure in concrete executor: ${concreteExecutionResult.result.exceptionOrNull()}" }
+                return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
+            }
+
             if (concreteExecutionResult.violatesUtMockAssumption()) {
                 logger.debug { "Generated test case by fuzzer violates the UtMock assumption: $concreteExecutionResult" }
                 return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
@@ -559,14 +572,16 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId
 private suspend fun ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>.executeConcretely(
     methodUnderTest: ExecutableId,
     stateBefore: EnvironmentModels,
-    instrumentation: List<UtInstrumentation>
+    instrumentation: List<UtInstrumentation>,
+    specification: UtConcreteExecutionSpecification? = null,
 ): UtConcreteExecutionResult = executeAsync(
     methodUnderTest.classId.name,
     methodUnderTest.signature,
     arrayOf(),
     parameters = UtConcreteExecutionData(
         stateBefore,
-        instrumentation
+        instrumentation,
+        specification = specification
     )
 ).convertToAssemble(methodUnderTest.classId.packageName)
 
@@ -623,3 +638,7 @@ private fun UtConcreteExecutionResult.violatesUtMockAssumption(): Boolean {
     // so we can't cast them to each other.
     return result.exceptionOrNull()?.javaClass?.name == UtMockAssumptionViolatedException::class.java.name
 }
+
+private fun UtConcreteExecutionResult.isExpectedFailure(): Boolean {
+    return result is UtIgnoreFailure
+}

utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

@@ -10,8 +10,8 @@ import org.utbot.framework.codegen.domain.NoStaticMocking
 import org.utbot.framework.codegen.domain.StaticsMocking
 import org.utbot.framework.codegen.domain.TestFramework
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionData
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionData
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionResult
 import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.CodegenLanguage

utbot-framework/src/main/kotlin/org/utbot/framework/util/UtConcreteExecutionResultUtils.kt

@@ -4,7 +4,7 @@ import org.utbot.framework.assemble.AssembleModelGenerator
 import org.utbot.framework.plugin.api.EnvironmentModels
 import org.utbot.framework.plugin.api.UtExecutionSuccess
 import org.utbot.framework.plugin.api.UtModel
-import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionResult
 import java.util.IdentityHashMap
 
 private fun UtConcreteExecutionResult.updateWithAssembleModels(

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/UtExecutionInstrumentation.kt

@@ -3,53 +3,24 @@ package org.utbot.instrumentation.instrumentation.execution
 import java.security.ProtectionDomain
 import java.util.IdentityHashMap
 import kotlin.reflect.jvm.javaMethod
-import org.utbot.framework.UtSettings
-import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy
-import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
-import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext
-import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController
-import org.utbot.instrumentation.instrumentation.execution.phases.start
-import org.utbot.framework.plugin.api.Coverage
 import org.utbot.framework.plugin.api.EnvironmentModels
 import org.utbot.framework.plugin.api.FieldId
-import org.utbot.framework.plugin.api.UtExecutionResult
-import org.utbot.framework.plugin.api.UtInstrumentation
 import org.utbot.framework.plugin.api.UtModel
 import org.utbot.framework.plugin.api.util.singleExecutableId
 import org.utbot.instrumentation.instrumentation.ArgumentList
 import org.utbot.instrumentation.instrumentation.Instrumentation
 import org.utbot.instrumentation.instrumentation.InvokeInstrumentation
 import org.utbot.instrumentation.instrumentation.et.TraceHandler
+import org.utbot.instrumentation.instrumentation.execution.constructors.ConstructOnlyUserClassesOrCachedObjectsStrategy
+import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionData
+import org.utbot.instrumentation.instrumentation.execution.data.UtConcreteExecutionResult
+import org.utbot.instrumentation.instrumentation.execution.mock.InstrumentationContext
+import org.utbot.instrumentation.instrumentation.execution.phases.PhasesController
+import org.utbot.instrumentation.instrumentation.execution.phases.start
 import org.utbot.instrumentation.instrumentation.instrumenter.Instrumenter
 import org.utbot.instrumentation.instrumentation.mock.MockClassVisitor
 
-/**
- * Consists of the data needed to execute the method concretely. Also includes method arguments stored in models.
- *
- * @property [stateBefore] is necessary for construction of parameters of a concrete call.
- * @property [instrumentation] is necessary for mocking static methods and new instances.
- * @property [timeout] is timeout for specific concrete execution (in milliseconds).
- * By default is initialized from [UtSettings.concreteExecutionTimeoutInInstrumentedProcess]
- */
-data class UtConcreteExecutionData(
-    val stateBefore: EnvironmentModels,
-    val instrumentation: List<UtInstrumentation>,
-    val timeout: Long = UtSettings.concreteExecutionTimeoutInInstrumentedProcess
-)
-
-class UtConcreteExecutionResult(
-    val stateAfter: EnvironmentModels,
-    val result: UtExecutionResult,
-    val coverage: Coverage
-) {
-    override fun toString(): String = buildString {
-        appendLine("UtConcreteExecutionResult(")
-        appendLine("stateAfter=$stateAfter")
-        appendLine("result=$result")
-        appendLine("coverage=$coverage)")
-    }
-}
-
 object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
     private val delegateInstrumentation = InvokeInstrumentation()
 
@@ -78,15 +49,16 @@ object UtExecutionInstrumentation : Instrumentation<UtConcreteExecutionResult> {
         if (parameters !is UtConcreteExecutionData) {
             throw IllegalArgumentException("Argument parameters must be of type UtConcreteExecutionData, but was: ${parameters?.javaClass}")
         }
-        val (stateBefore, instrumentations, timeout) = parameters // smart cast to UtConcreteExecutionData
+        val (stateBefore, instrumentations, timeout, specification) = parameters // smart cast to UtConcreteExecutionData
 
         val methodId = clazz.singleExecutableId(methodSignature)
         val returnClassId = methodId.returnType
 
         return PhasesController(
             instrumentationContext,
             traceHandler,
-            delegateInstrumentation
+            delegateInstrumentation,
+            specification,
         ).computeConcreteExecutionResult {
             // construction
             val (params, statics, cache) = valueConstructionContext.start {

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/constructors/MockValueConstructor.kt

@@ -53,6 +53,7 @@ import org.utbot.framework.plugin.api.util.jField
 import org.utbot.framework.plugin.api.util.method
 import org.utbot.framework.plugin.api.util.utContext
 import org.utbot.framework.plugin.api.util.anyInstance
+import org.utbot.instrumentation.instrumentation.execution.phases.withForceException
 import org.utbot.instrumentation.process.runSandbox
 
 /**
@@ -234,7 +235,10 @@ class MockValueConstructor(
     }
 
     private fun generateMockitoMock(clazz: Class<*>, mocks: Map<ExecutableId, List<UtModel>>): Any {
-        return Mockito.mock(clazz, generateMockitoAnswer(mocks))
+        val answer = generateMockitoAnswer(mocks)
+        return withForceException {
+            Mockito.mock(clazz, answer)
+        }
     }
 
     private fun computeConcreteValuesForMethods(
@@ -261,13 +265,15 @@ class MockValueConstructor(
             if (method !is MethodId) {
                 throw IllegalArgumentException("Expected MethodId, but got: $method")
             }
-            MethodMockController(
-                method.classId.jClass,
-                method.method,
-                instance,
-                values,
-                instrumentationContext
-            )
+            withForceException {
+                MethodMockController(
+                    method.classId.jClass,
+                    method.method,
+                    instance,
+                    values,
+                    instrumentationContext
+                )
+            }
         }
 
     }
@@ -289,11 +295,13 @@ class MockValueConstructor(
         instrumentations: List<UtNewInstanceInstrumentation>,
     ) {
         controllers += instrumentations.map { mock ->
-            InstanceMockController(
-                mock.classId,
-                mock.instances.map { mockAndGet(it) },
-                mock.callSites.map { Type.getType(it.jClass).internalName }.toSet()
-            )
+            withForceException {
+                InstanceMockController(
+                    mock.classId,
+                    mock.instances.map { mockAndGet(it) },
+                    mock.callSites.map { Type.getType(it.jClass).internalName }.toSet()
+                )
+            }
         }
     }
 
@@ -393,7 +401,9 @@ class MockValueConstructor(
             val capturedArguments = lambdaModel.capturedValues
                 .map { model -> CapturedArgument(type = model.classId.jClass, value = value(model)) }
                 .toTypedArray()
-            constructStaticLambda(samType, declaringClass, lambdaName, *capturedArguments)
+            withForceException {
+                constructStaticLambda(samType, declaringClass, lambdaName, *capturedArguments)
+            }
         } else {
             val capturedReceiverModel = lambdaModel.capturedValues.firstOrNull()
                 ?: error("Non-static lambda must capture `this` instance, so there must be at least one captured value")
@@ -403,7 +413,9 @@ class MockValueConstructor(
             val capturedArguments = lambdaModel.capturedValues.subList(1, lambdaModel.capturedValues.size)
                 .map { model -> CapturedArgument(type = model.classId.jClass, value = value(model)) }
                 .toTypedArray()
-            constructLambda(samType, declaringClass, lambdaName, capturedReceiver, *capturedArguments)
+            withForceException {
+                constructLambda(samType, declaringClass, lambdaName, capturedReceiver, *capturedArguments)
+            }
         }
         constructedObjects[lambdaModel] = lambda
         return lambda
@@ -495,7 +507,7 @@ class MockValueConstructor(
 
     private fun javaClass(id: ClassId) = kClass(id).java
 
-    private fun kClass(id: ClassId) =
+    private fun kClass(id: ClassId) = withForceException {
         if (id.elementClassId != null) {
             arrayClassOf(id.elementClassId!!)
         } else {
@@ -511,8 +523,9 @@ class MockValueConstructor(
                 else -> classLoader.loadClass(id.name).kotlin
             }
         }
+    }
 
-    private fun arrayClassOf(elementClassId: ClassId): KClass<*> =
+    private fun arrayClassOf(elementClassId: ClassId): KClass<*> = withForceException {
         if (elementClassId.elementClassId != null) {
             val elementClass = arrayClassOf(elementClassId.elementClassId!!)
             java.lang.reflect.Array.newInstance(elementClass.java, 0)::class
@@ -532,6 +545,7 @@ class MockValueConstructor(
                 }
             }
         }
+    }
 
     override fun close() {
         controllers.forEach { it.close() }

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/data/UtConcreteExecutionData.kt

@@ -0,0 +1,21 @@
+package org.utbot.instrumentation.instrumentation.execution.data
+
+import org.utbot.framework.UtSettings
+import org.utbot.framework.plugin.api.EnvironmentModels
+import org.utbot.framework.plugin.api.UtInstrumentation
+
+/**
+ * Consists of the data needed to execute the method concretely. Also includes method arguments stored in models.
+ *
+ * @property [stateBefore] is necessary for construction of parameters of a concrete call.
+ * @property [instrumentation] is necessary for mocking static methods and new instances.
+ * @property [timeout] is timeout for specific concrete execution (in milliseconds).
+ * @property [specification] is strategy for controlling of results handling and validation (null means "without any validation").
+ * By default, is initialized from [UtSettings.concreteExecutionTimeoutInInstrumentedProcess]
+ */
+data class UtConcreteExecutionData(
+    val stateBefore: EnvironmentModels,
+    val instrumentation: List<UtInstrumentation>,
+    val timeout: Long = UtSettings.concreteExecutionTimeoutInInstrumentedProcess,
+    val specification: UtConcreteExecutionSpecification? = null,
+)

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/data/UtConcreteExecutionResult.kt

@@ -0,0 +1,18 @@
+package org.utbot.instrumentation.instrumentation.execution.data
+
+import org.utbot.framework.plugin.api.Coverage
+import org.utbot.framework.plugin.api.EnvironmentModels
+import org.utbot.framework.plugin.api.UtExecutionResult
+
+class UtConcreteExecutionResult(
+    val stateAfter: EnvironmentModels,
+    val result: UtExecutionResult,
+    val coverage: Coverage
+) {
+    override fun toString(): String = buildString {
+        appendLine("UtConcreteExecutionResult(")
+        appendLine("stateAfter=$stateAfter")
+        appendLine("result=$result")
+        appendLine("coverage=$coverage)")
+    }
+}

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/data/UtConcreteExecutionSpecification.kt

@@ -0,0 +1,24 @@
+package org.utbot.instrumentation.instrumentation.execution.data
+
+import org.utbot.framework.plugin.api.UtIgnoreFailure
+import org.utbot.instrumentation.instrumentation.execution.phases.ValueConstructionPhaseError
+
+/**
+ * Specification controls result handling and validation.
+ */
+interface UtConcreteExecutionSpecification {
+
+    /**
+     * If an exception is expected, it will be handling and translate to [UtIgnoreFailure]
+     */
+    fun exceptionIsExpected(exception: Exception): Boolean
+}
+
+/**
+ * Fuzzing specification allows to ignore errors during values construction.
+ */
+class FuzzingSpecification : UtConcreteExecutionSpecification {
+    override fun exceptionIsExpected(exception: Exception): Boolean {
+        return exception is ValueConstructionPhaseError
+    }
+}

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/instrumentation/execution/phases/PhaseContext.kt

@@ -5,9 +5,11 @@ interface PhaseContext<E: PhaseError> {
     fun wrapError(error: Throwable): E
 }
 
-inline fun <reified R, T: PhaseContext<*>> T.start(block: T.() -> R): R =
+internal inline fun <reified R, T: PhaseContext<*>> T.start(block: T.() -> R): R =
     try {
         block()
+    } catch (e: UtConcreteExecutionForceException) {
+        throw UtConcreteExecutionForceException(wrapError(e.cause))
     } catch (e: Throwable) {
         throw wrapError(e)
     }