use experimental oracles

core-it/src/test/kotlin/org/evomaster/core/problem/rest/nonworkingdelete/NonWorkingDeleteTest.kt

@@ -30,6 +30,7 @@ class NonWorkingDeleteTest: IntegrationTestRestBase() {
         getEMConfig().security = false
         getEMConfig().schemaOracles = false
         getEMConfig().httpOracles = true
+        getEMConfig().useExperimentalOracles = true
     }
 
 

core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/ForgottenAuthenticationTest.kt

@@ -35,6 +35,7 @@ class ForgottenAuthenticationTest: IntegrationTestRestBase()  {
         ForgottenAuthenticationApplication.reset()
         getEMConfig().security = true
         getEMConfig().schemaOracles = false
+        getEMConfig().useExperimentalOracles = true
     }
 
     @Test

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -2575,6 +2575,11 @@ class EMConfig {
     )
     var disabledOracleCodes = ""
 
+    @Cfg("Enables experimental oracles. When true, ExperimentalFaultCategory items are included alongside standard ones. " +
+            "Experimental oracles may be unstable or unverified and should only be used for testing or evaluation purposes. " +
+            "When false, all experimental oracles are disabled.")
+    var useExperimentalOracles = false
+
     enum class VulnerableInputClassificationStrategy {
         /**
          * Uses the manual methods to select the vulnerable inputs.
@@ -2867,28 +2872,79 @@ class EMConfig {
      */
     var gaSolutionSource: GASolutionSource = GASolutionSource.ARCHIVE
 
-    private var disabledOracleCodesList: List<FaultCategory>? = null
 
-    fun getDisabledOracleCodesList(): List<FaultCategory> {
-        if (disabledOracleCodesList == null) {
-            disabledOracleCodesList = disabledOracleCodes
-                .split(",")
-                .mapNotNull { it.trim().takeIf { s -> s.isNotEmpty() } }
-                .map { str ->
-                    val code = str.toIntOrNull()
-                        ?: throw ConfigProblemException("Invalid number: $str")
-
-                    val allCategories = DefinedFaultCategory.values().asList() +
-                            ExperimentalFaultCategory.values()
-
-                    allCategories.firstOrNull { it.code == code }
-                        ?: throw ConfigProblemException(
-                            "Invalid fault code: $code" +
-                                    " All available codes are: \n" +
-                                    allCategories.joinToString("\n") { "${it.code} (${it.name})" }
-                        )
+    /**
+     * Not all oracles are active by default.
+     * Some might be experimental, while others might be explicitly excluded by the user
+     */
+    fun isEnabledFaultCategory(category: FaultCategory) : Boolean{
+        return category !in getDisabledOracleCodesList()
+    }
+
+    private fun isFaultCodeActive(
+        code: Int,
+        disabledCodes: Set<Int>
+    ): Boolean {
+        val isExperimental = ExperimentalFaultCategory.entries.any { it.code == code }
+        if (isExperimental && !useExperimentalOracles) return false
+        if (code in disabledCodes) return false
+        return true
+    }
+
+    private fun parseDisabledCodesOrThrow(
+        disabledOracleCodes: String,
+    ): Set<Int> {
+        if (disabledOracleCodes.isBlank()) return emptySet()
+
+        val tokens = disabledOracleCodes
+            .split(",")
+            .mapNotNull { it.trim().takeIf { s -> s.isNotEmpty() } }
+
+        val codes = tokens.map { token ->
+            token.toIntOrNull() ?: throw ConfigProblemException("Invalid number: $token")
+        }
+
+        val definedCodes = DefinedFaultCategory.entries.map { it.code }.toSet()
+        val experimentalCodes = ExperimentalFaultCategory.entries.map { it.code }.toSet()
+        val knownCodes = definedCodes + experimentalCodes
+
+        val unknown = codes.filter { it !in knownCodes }
+        if (unknown.isNotEmpty()) {
+            val message = buildString {
+                appendLine("Invalid fault code(s): ${unknown.joinToString(", ")}")
+                appendLine("All available defined codes:")
+                appendLine(DefinedFaultCategory.entries.joinToString("\n") { "${it.code} (${it.name})" })
+                appendLine("All available experimental codes:")
+                appendLine(ExperimentalFaultCategory.entries.joinToString("\n") { "${it.code} (${it.name})" })
+                if (!useExperimentalOracles) {
+                    appendLine("Note: Experimental oracles are currently disabled (useExperimentalOracles=false).")
                 }
+            }
+            throw ConfigProblemException(message)
         }
+
+        return codes.toSet()
+    }
+
+    private var disabledOracleCodesList: List<FaultCategory>? = null
+
+    private fun getDisabledOracleCodesList(): List<FaultCategory> {
+        if (disabledOracleCodesList != null) {
+            return disabledOracleCodesList!!
+        }
+
+        val definedCategories = DefinedFaultCategory.entries
+        val experimentalCategories = ExperimentalFaultCategory.entries
+
+        val allCategories: List<FaultCategory> = definedCategories + experimentalCategories
+
+        val userDisabledCodes: Set<Int> = parseDisabledCodesOrThrow(disabledOracleCodes)
+
+        val disabled: List<FaultCategory> = allCategories.filter { category ->
+            !isFaultCodeActive(category.code, userDisabledCodes)
+        }
+
+        disabledOracleCodesList = disabled.distinct()
         return disabledOracleCodesList!!
     }
 

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -420,14 +420,13 @@ class Main {
                     val securityRest = injector.getInstance(SecurityRest::class.java)
                     val solution = securityRest.applySecurityPhase()
 
-                    if (config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) {
+                    if (config.ssrf && config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) {
                         LoggingUtil.getInfoLogger().info("Starting to apply SSRF detection.")
 
                         val ssrfAnalyser = injector.getInstance(SSRFAnalyser::class.java)
                         ssrfAnalyser.apply()
                     } else {
-                        if(DefinedFaultCategory.SSRF in config.getDisabledOracleCodesList())
-                        {
+                        if(!config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) {
                             LoggingUtil.uniqueUserInfo("Skipping security test for SSRF detection as disabled in configuration")
                         }
 

core/src/main/kotlin/org/evomaster/core/problem/enterprise/DetectedFaultUtils.kt

@@ -1,6 +1,7 @@
 package org.evomaster.core.problem.enterprise
 
 import com.webfuzzing.commons.faults.FaultCategory
+import org.evomaster.core.EMConfig
 import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.Solution
 import org.evomaster.core.search.action.ActionResult
@@ -38,15 +39,15 @@ object DetectedFaultUtils {
             .toSet()
     }
 
-    fun verifyExcludedCategories(ei: EvaluatedIndividual<*>, excludedCategories: List<FaultCategory>) : Boolean {
+    fun verifyExcludedCategories(ei: EvaluatedIndividual<*>, config: EMConfig) : Boolean {
 
         // if not an enterprise individual, then no need to check
         if(ei.individual !is EnterpriseIndividual){
             return true
         }
 
         val detected = getDetectedFaultCategories(ei)
-        return excludedCategories.intersect(detected).isEmpty()
+        return detected.all{config.isEnabledFaultCategory(it)}
     }
 
 }

core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLFitness.kt

@@ -223,7 +223,7 @@ open class GraphQLFitness : HttpWsFitness<GraphQLIndividual>() {
         }
 
         if (status == 500) {
-            if (DefinedFaultCategory.HTTP_STATUS_500 !in config.getDisabledOracleCodesList()) {
+            if (config.isEnabledFaultCategory(DefinedFaultCategory.HTTP_STATUS_500)) {
                 Lazy.assert { location5xx != null || config.blackBox }
                 /*
                     500 codes "might" be bugs. To distinguish between different bugs

core/src/main/kotlin/org/evomaster/core/problem/rest/service/SecurityRest.kt

@@ -253,7 +253,7 @@ class SecurityRest {
 
     private fun accessControlBasedOnRESTGuidelines() {
 
-        if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_WRONG_AUTHORIZATION)){
+        if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_WRONG_AUTHORIZATION)){
             LoggingUtil.uniqueUserInfo("Skipping security test for forbidden but ok others as disabled in configuration")
         } else {
             // quite a few rules here that can be defined
@@ -262,21 +262,21 @@ class SecurityRest {
             handleForbiddenOperationButOKOthers(HttpVerb.PATCH)
         }
 
-        if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_EXISTENCE_LEAKAGE)){
+        if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_EXISTENCE_LEAKAGE)){
             LoggingUtil.uniqueUserInfo("Skipping security test for existence leakage as disabled in configuration")
         } else {
             // getting 404 instead of 403
             handleExistenceLeakage()
         }
 
-        if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_NOT_RECOGNIZED_AUTHENTICATED)){
+        if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_NOT_RECOGNIZED_AUTHENTICATED)){
             LoggingUtil.uniqueUserInfo("Skipping security test for not recognized authenticated as disabled in configuration")
         } else {
             //authenticated, but wrongly getting 401 (eg instead of 403)
             handleNotRecognizedAuthenticated()
         }
 
-        if(config.getDisabledOracleCodesList().contains(ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION)) {
+        if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION)) {
             LoggingUtil.uniqueUserInfo("Skipping experimental security test for forgotten authentication as disabled in configuration")
         } else {
             handleForgottenAuthentication()

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt

@@ -544,7 +544,7 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
         }
 
         if (status == 500){
-            if( DefinedFaultCategory.HTTP_STATUS_500 !in config.getDisabledOracleCodesList()) {
+            if( config.isEnabledFaultCategory(DefinedFaultCategory.HTTP_STATUS_500)) {
                 /*
                     500 codes "might" be bugs. To distinguish between different bugs
                     that crash the same endpoint, we need to know what was the last
@@ -733,7 +733,7 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
             }
         }
 
-        if(DefinedFaultCategory.SCHEMA_INVALID_RESPONSE !in config.getDisabledOracleCodesList()){
+        if(config.isEnabledFaultCategory(DefinedFaultCategory.SCHEMA_INVALID_RESPONSE)){
             handleSchemaOracles(a, rcr, fv)
         } else {
             LoggingUtil.uniqueUserInfo("Schema oracles disabled via configuration")
@@ -745,7 +745,7 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
             responseClassifier.updateModel(a, rcr)
         }
 
-        if (config.security && config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) {
+        if (config.security && config.ssrf && config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) {
             if (ssrfAnalyser.anyCallsMadeToHTTPVerifier(a)) {
                 rcr.setVulnerableForSSRF(true)
             }
@@ -1124,7 +1124,7 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
             analyzeSecurityProperties(individual,actionResults,fv)
         }
 
-        if (config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) {
+        if (config.ssrf &&  config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) {
             handleSsrfFaults(individual, actionResults, fv)
         }
 
@@ -1136,9 +1136,17 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
     }
 
     private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List<ActionResult>, fv: FitnessValue) {
+        if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_NONWORKING_DELETE)) {
+            LoggingUtil.uniqueUserInfo("Skipping experimental security test for non-working DELETE, as it has been disabled via configuration")
+        } else {
+            handleDeleteShouldDelete(individual, actionResults, fv)
+        }
 
-        handleDeleteShouldDelete(individual, actionResults, fv)
-        handleRepeatedCreatePut(individual, actionResults, fv)
+        if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_REPEATED_CREATE_PUT)) {
+            LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration")
+        } else {
+            handleRepeatedCreatePut(individual, actionResults, fv)
+        }
     }
 
     private fun handleRepeatedCreatePut(

core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt

@@ -101,11 +101,7 @@ abstract class FitnessFunction<T>  where T : Individual {
 //        }
 
         // check that excluded fault categories are not present
-        Lazy.assert{
-            DetectedFaultUtils.verifyExcludedCategories(ei as EvaluatedIndividual<Individual>,
-                config.getDisabledOracleCodesList() as List<FaultCategory>
-            )
-        }
+        Lazy.assert{ DetectedFaultUtils.verifyExcludedCategories(ei as EvaluatedIndividual<Individual>, config) }
 
         return ei
     }

docs/options.md

@@ -219,6 +219,7 @@ There are 3 types of options:
 |`testSuiteSplitType`| __Enum__. Instead of generating a single test file, it could be split in several files, according to different strategies. *Valid values*: `NONE, FAULTS`. *Default value*: `FAULTS`.|
 |`tournamentSize`| __Int__. Number of elements to consider in a Tournament Selection (if any is used in the search algorithm). *Constraints*: `min=1.0`. *Default value*: `10`.|
 |`treeDepth`| __Int__. Maximum tree depth in mutations/queries to be evaluated. This is to avoid issues when dealing with huge graphs in GraphQL. *Constraints*: `min=1.0`. *Default value*: `4`.|
+|`useExperimentalOracles`| __Boolean__. Enables experimental oracles. When true, ExperimentalFaultCategory items are included alongside standard ones. Experimental oracles may be unstable or unverified and should only be used for testing or evaluation purposes. When false, all experimental oracles are disabled. *Default value*: `false`.|
 |`useExtraSqlDbConstraintsProbability`| __Double__. Whether to analyze how SQL databases are accessed to infer extra constraints from the business logic. An example is javax/jakarta annotation constraints defined on JPA entities. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.|
 |`useMethodReplacement`| __Boolean__. Apply method replacement heuristics to smooth the search landscape. Note that the method replacement instrumentations would still be applied, it is just that their testing targets will be ignored in the fitness function if this option is set to false. *Default value*: `true`.|
 |`useNonIntegerReplacement`| __Boolean__. Apply non-integer numeric comparison heuristics to smooth the search landscape. *Default value*: `true`.|

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/delete/HttpOracleDeleteEMTest.kt

@@ -33,6 +33,7 @@ class HttpOracleDeleteEMTest : SpringTestBase(){
             setOption(args, "security", "false")
             setOption(args, "schemaOracles", "false")
             setOption(args, "httpOracles", "true")
+            setOption(args, "useExperimentalOracles", "true")
 
             val solution = initAndRun(args)
 

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/repeatedput/HttpOracleRepeatedPutEMTest.kt

@@ -33,6 +33,7 @@ class HttpOracleRepeatedPutEMTest : SpringTestBase(){
             setOption(args, "security", "false")
             setOption(args, "schemaOracles", "false")
             setOption(args, "httpOracles", "true")
+            setOption(args, "useExperimentalOracles", "true")
 
             val solution = initAndRun(args)
 

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt

@@ -30,6 +30,7 @@ class ForgottenAuthenticationDisableEMTest : SpringTestBase(){
 
             setOption(args, "security", "true")
             setOption(args, "schemaOracles", "false")
+            setOption(args, "useExperimentalOracles", "true")
             setOption(args, "disabledOracleCodes", ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION.code.toString())
 
             val solution = initAndRun(args)

e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt

@@ -32,6 +32,7 @@ class ForgottenAuthenticationEMTest : SpringTestBase(){
 
             setOption(args, "security", "true")
             setOption(args, "schemaOracles", "false")
+            setOption(args, "useExperimentalOracles", "true")
 
             val solution = initAndRun(args)