diff --git a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/nonworkingdelete/NonWorkingDeleteTest.kt b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/nonworkingdelete/NonWorkingDeleteTest.kt index a73e8d760f..49f96fbf4d 100644 --- a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/nonworkingdelete/NonWorkingDeleteTest.kt +++ b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/nonworkingdelete/NonWorkingDeleteTest.kt @@ -30,6 +30,7 @@ class NonWorkingDeleteTest: IntegrationTestRestBase() { getEMConfig().security = false getEMConfig().schemaOracles = false getEMConfig().httpOracles = true + getEMConfig().useExperimentalOracles = true } diff --git a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/ForgottenAuthenticationTest.kt b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/ForgottenAuthenticationTest.kt index 194e9eaf39..7fd437b58d 100644 --- a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/ForgottenAuthenticationTest.kt +++ b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/ForgottenAuthenticationTest.kt @@ -35,6 +35,7 @@ class ForgottenAuthenticationTest: IntegrationTestRestBase() { ForgottenAuthenticationApplication.reset() getEMConfig().security = true getEMConfig().schemaOracles = false + getEMConfig().useExperimentalOracles = true } @Test diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index cef40ae97a..8a2d0e744d 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2575,6 +2575,11 @@ class EMConfig { ) var disabledOracleCodes = "" + @Cfg("Enables experimental oracles. When true, ExperimentalFaultCategory items are included alongside standard ones. " + + "Experimental oracles may be unstable or unverified and should only be used for testing or evaluation purposes. " + + "When false, all experimental oracles are disabled.") + var useExperimentalOracles = false + enum class VulnerableInputClassificationStrategy { /** * Uses the manual methods to select the vulnerable inputs. @@ -2867,28 +2872,79 @@ class EMConfig { */ var gaSolutionSource: GASolutionSource = GASolutionSource.ARCHIVE - private var disabledOracleCodesList: List? = null - fun getDisabledOracleCodesList(): List { - if (disabledOracleCodesList == null) { - disabledOracleCodesList = disabledOracleCodes - .split(",") - .mapNotNull { it.trim().takeIf { s -> s.isNotEmpty() } } - .map { str -> - val code = str.toIntOrNull() - ?: throw ConfigProblemException("Invalid number: $str") - - val allCategories = DefinedFaultCategory.values().asList() + - ExperimentalFaultCategory.values() - - allCategories.firstOrNull { it.code == code } - ?: throw ConfigProblemException( - "Invalid fault code: $code" + - " All available codes are: \n" + - allCategories.joinToString("\n") { "${it.code} (${it.name})" } - ) + /** + * Not all oracles are active by default. + * Some might be experimental, while others might be explicitly excluded by the user + */ + fun isEnabledFaultCategory(category: FaultCategory) : Boolean{ + return category !in getDisabledOracleCodesList() + } + + private fun isFaultCodeActive( + code: Int, + disabledCodes: Set + ): Boolean { + val isExperimental = ExperimentalFaultCategory.entries.any { it.code == code } + if (isExperimental && !useExperimentalOracles) return false + if (code in disabledCodes) return false + return true + } + + private fun parseDisabledCodesOrThrow( + disabledOracleCodes: String, + ): Set { + if (disabledOracleCodes.isBlank()) return emptySet() + + val tokens = disabledOracleCodes + .split(",") + .mapNotNull { it.trim().takeIf { s -> s.isNotEmpty() } } + + val codes = tokens.map { token -> + token.toIntOrNull() ?: throw ConfigProblemException("Invalid number: $token") + } + + val definedCodes = DefinedFaultCategory.entries.map { it.code }.toSet() + val experimentalCodes = ExperimentalFaultCategory.entries.map { it.code }.toSet() + val knownCodes = definedCodes + experimentalCodes + + val unknown = codes.filter { it !in knownCodes } + if (unknown.isNotEmpty()) { + val message = buildString { + appendLine("Invalid fault code(s): ${unknown.joinToString(", ")}") + appendLine("All available defined codes:") + appendLine(DefinedFaultCategory.entries.joinToString("\n") { "${it.code} (${it.name})" }) + appendLine("All available experimental codes:") + appendLine(ExperimentalFaultCategory.entries.joinToString("\n") { "${it.code} (${it.name})" }) + if (!useExperimentalOracles) { + appendLine("Note: Experimental oracles are currently disabled (useExperimentalOracles=false).") } + } + throw ConfigProblemException(message) } + + return codes.toSet() + } + + private var disabledOracleCodesList: List? = null + + private fun getDisabledOracleCodesList(): List { + if (disabledOracleCodesList != null) { + return disabledOracleCodesList!! + } + + val definedCategories = DefinedFaultCategory.entries + val experimentalCategories = ExperimentalFaultCategory.entries + + val allCategories: List = definedCategories + experimentalCategories + + val userDisabledCodes: Set = parseDisabledCodesOrThrow(disabledOracleCodes) + + val disabled: List = allCategories.filter { category -> + !isFaultCodeActive(category.code, userDisabledCodes) + } + + disabledOracleCodesList = disabled.distinct() return disabledOracleCodesList!! } diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index c3f6e1a228..f8705bd650 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -420,14 +420,13 @@ class Main { val securityRest = injector.getInstance(SecurityRest::class.java) val solution = securityRest.applySecurityPhase() - if (config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) { + if (config.ssrf && config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) { LoggingUtil.getInfoLogger().info("Starting to apply SSRF detection.") val ssrfAnalyser = injector.getInstance(SSRFAnalyser::class.java) ssrfAnalyser.apply() } else { - if(DefinedFaultCategory.SSRF in config.getDisabledOracleCodesList()) - { + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) { LoggingUtil.uniqueUserInfo("Skipping security test for SSRF detection as disabled in configuration") } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/DetectedFaultUtils.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/DetectedFaultUtils.kt index 746ee683b2..7174fea60e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/DetectedFaultUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/DetectedFaultUtils.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.enterprise import com.webfuzzing.commons.faults.FaultCategory +import org.evomaster.core.EMConfig import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution import org.evomaster.core.search.action.ActionResult @@ -38,7 +39,7 @@ object DetectedFaultUtils { .toSet() } - fun verifyExcludedCategories(ei: EvaluatedIndividual<*>, excludedCategories: List) : Boolean { + fun verifyExcludedCategories(ei: EvaluatedIndividual<*>, config: EMConfig) : Boolean { // if not an enterprise individual, then no need to check if(ei.individual !is EnterpriseIndividual){ @@ -46,7 +47,7 @@ object DetectedFaultUtils { } val detected = getDetectedFaultCategories(ei) - return excludedCategories.intersect(detected).isEmpty() + return detected.all{config.isEnabledFaultCategory(it)} } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLFitness.kt index 72dce7c0bf..cdf6927905 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLFitness.kt @@ -223,7 +223,7 @@ open class GraphQLFitness : HttpWsFitness() { } if (status == 500) { - if (DefinedFaultCategory.HTTP_STATUS_500 !in config.getDisabledOracleCodesList()) { + if (config.isEnabledFaultCategory(DefinedFaultCategory.HTTP_STATUS_500)) { Lazy.assert { location5xx != null || config.blackBox } /* 500 codes "might" be bugs. To distinguish between different bugs diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/SecurityRest.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/SecurityRest.kt index ac8c3bc12c..fad8764c8a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/SecurityRest.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/SecurityRest.kt @@ -253,7 +253,7 @@ class SecurityRest { private fun accessControlBasedOnRESTGuidelines() { - if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_WRONG_AUTHORIZATION)){ + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_WRONG_AUTHORIZATION)){ LoggingUtil.uniqueUserInfo("Skipping security test for forbidden but ok others as disabled in configuration") } else { // quite a few rules here that can be defined @@ -262,21 +262,21 @@ class SecurityRest { handleForbiddenOperationButOKOthers(HttpVerb.PATCH) } - if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_EXISTENCE_LEAKAGE)){ + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_EXISTENCE_LEAKAGE)){ LoggingUtil.uniqueUserInfo("Skipping security test for existence leakage as disabled in configuration") } else { // getting 404 instead of 403 handleExistenceLeakage() } - if(config.getDisabledOracleCodesList().contains(DefinedFaultCategory.SECURITY_NOT_RECOGNIZED_AUTHENTICATED)){ + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_NOT_RECOGNIZED_AUTHENTICATED)){ LoggingUtil.uniqueUserInfo("Skipping security test for not recognized authenticated as disabled in configuration") } else { //authenticated, but wrongly getting 401 (eg instead of 403) handleNotRecognizedAuthenticated() } - if(config.getDisabledOracleCodesList().contains(ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION)) { + if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION)) { LoggingUtil.uniqueUserInfo("Skipping experimental security test for forgotten authentication as disabled in configuration") } else { handleForgottenAuthentication() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 3737c39ad7..163d7d8e93 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -544,7 +544,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { } if (status == 500){ - if( DefinedFaultCategory.HTTP_STATUS_500 !in config.getDisabledOracleCodesList()) { + if( config.isEnabledFaultCategory(DefinedFaultCategory.HTTP_STATUS_500)) { /* 500 codes "might" be bugs. To distinguish between different bugs that crash the same endpoint, we need to know what was the last @@ -733,7 +733,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { } } - if(DefinedFaultCategory.SCHEMA_INVALID_RESPONSE !in config.getDisabledOracleCodesList()){ + if(config.isEnabledFaultCategory(DefinedFaultCategory.SCHEMA_INVALID_RESPONSE)){ handleSchemaOracles(a, rcr, fv) } else { LoggingUtil.uniqueUserInfo("Schema oracles disabled via configuration") @@ -745,7 +745,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { responseClassifier.updateModel(a, rcr) } - if (config.security && config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) { + if (config.security && config.ssrf && config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) { if (ssrfAnalyser.anyCallsMadeToHTTPVerifier(a)) { rcr.setVulnerableForSSRF(true) } @@ -1124,7 +1124,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { analyzeSecurityProperties(individual,actionResults,fv) } - if (config.ssrf && DefinedFaultCategory.SSRF !in config.getDisabledOracleCodesList()) { + if (config.ssrf && config.isEnabledFaultCategory(DefinedFaultCategory.SSRF)) { handleSsrfFaults(individual, actionResults, fv) } @@ -1136,9 +1136,17 @@ abstract class AbstractRestFitness : HttpWsFitness() { } private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List, fv: FitnessValue) { + if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_NONWORKING_DELETE)) { + LoggingUtil.uniqueUserInfo("Skipping experimental security test for non-working DELETE, as it has been disabled via configuration") + } else { + handleDeleteShouldDelete(individual, actionResults, fv) + } - handleDeleteShouldDelete(individual, actionResults, fv) - handleRepeatedCreatePut(individual, actionResults, fv) + if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_REPEATED_CREATE_PUT)) { + LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration") + } else { + handleRepeatedCreatePut(individual, actionResults, fv) + } } private fun handleRepeatedCreatePut( diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt index fa23640831..08bb753189 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt @@ -101,11 +101,7 @@ abstract class FitnessFunction where T : Individual { // } // check that excluded fault categories are not present - Lazy.assert{ - DetectedFaultUtils.verifyExcludedCategories(ei as EvaluatedIndividual, - config.getDisabledOracleCodesList() as List - ) - } + Lazy.assert{ DetectedFaultUtils.verifyExcludedCategories(ei as EvaluatedIndividual, config) } return ei } diff --git a/docs/options.md b/docs/options.md index b4e871d5cd..4847bbc66e 100644 --- a/docs/options.md +++ b/docs/options.md @@ -219,6 +219,7 @@ There are 3 types of options: |`testSuiteSplitType`| __Enum__. Instead of generating a single test file, it could be split in several files, according to different strategies. *Valid values*: `NONE, FAULTS`. *Default value*: `FAULTS`.| |`tournamentSize`| __Int__. Number of elements to consider in a Tournament Selection (if any is used in the search algorithm). *Constraints*: `min=1.0`. *Default value*: `10`.| |`treeDepth`| __Int__. Maximum tree depth in mutations/queries to be evaluated. This is to avoid issues when dealing with huge graphs in GraphQL. *Constraints*: `min=1.0`. *Default value*: `4`.| +|`useExperimentalOracles`| __Boolean__. Enables experimental oracles. When true, ExperimentalFaultCategory items are included alongside standard ones. Experimental oracles may be unstable or unverified and should only be used for testing or evaluation purposes. When false, all experimental oracles are disabled. *Default value*: `false`.| |`useExtraSqlDbConstraintsProbability`| __Double__. Whether to analyze how SQL databases are accessed to infer extra constraints from the business logic. An example is javax/jakarta annotation constraints defined on JPA entities. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.9`.| |`useMethodReplacement`| __Boolean__. Apply method replacement heuristics to smooth the search landscape. Note that the method replacement instrumentations would still be applied, it is just that their testing targets will be ignored in the fitness function if this option is set to false. *Default value*: `true`.| |`useNonIntegerReplacement`| __Boolean__. Apply non-integer numeric comparison heuristics to smooth the search landscape. *Default value*: `true`.| diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/delete/HttpOracleDeleteEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/delete/HttpOracleDeleteEMTest.kt index adcc2de3e0..d6c655efb0 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/delete/HttpOracleDeleteEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/delete/HttpOracleDeleteEMTest.kt @@ -33,6 +33,7 @@ class HttpOracleDeleteEMTest : SpringTestBase(){ setOption(args, "security", "false") setOption(args, "schemaOracles", "false") setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") val solution = initAndRun(args) diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/repeatedput/HttpOracleRepeatedPutEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/repeatedput/HttpOracleRepeatedPutEMTest.kt index 93c58fb91d..7c958eda60 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/repeatedput/HttpOracleRepeatedPutEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/repeatedput/HttpOracleRepeatedPutEMTest.kt @@ -33,6 +33,7 @@ class HttpOracleRepeatedPutEMTest : SpringTestBase(){ setOption(args, "security", "false") setOption(args, "schemaOracles", "false") setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") val solution = initAndRun(args) diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt index b6138eb2ca..f75c334c44 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt @@ -30,6 +30,7 @@ class ForgottenAuthenticationDisableEMTest : SpringTestBase(){ setOption(args, "security", "true") setOption(args, "schemaOracles", "false") + setOption(args, "useExperimentalOracles", "true") setOption(args, "disabledOracleCodes", ExperimentalFaultCategory.SECURITY_FORGOTTEN_AUTHENTICATION.code.toString()) val solution = initAndRun(args) diff --git a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt index cb46e7398c..1559bcf61a 100644 --- a/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt +++ b/e2e-tests/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt @@ -32,6 +32,7 @@ class ForgottenAuthenticationEMTest : SpringTestBase(){ setOption(args, "security", "true") setOption(args, "schemaOracles", "false") + setOption(args, "useExperimentalOracles", "true") val solution = initAndRun(args)