diff --git a/core-tests/e2e-tests/spring/spring-rest-h2-z3solver/src/test/java/org/evomaster/e2etests/spring/h2/z3solver/Z3SolverEMTest.java b/core-tests/e2e-tests/spring/spring-rest-h2-z3solver/src/test/java/org/evomaster/e2etests/spring/h2/z3solver/Z3SolverEMTest.java index 339d6d16b4..e47ebd1e3d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-h2-z3solver/src/test/java/org/evomaster/e2etests/spring/h2/z3solver/Z3SolverEMTest.java +++ b/core-tests/e2e-tests/spring/spring-rest-h2-z3solver/src/test/java/org/evomaster/e2etests/spring/h2/z3solver/Z3SolverEMTest.java @@ -5,10 +5,13 @@ import org.evomaster.core.problem.rest.data.RestIndividual; import org.evomaster.core.search.Solution; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertFalse; + +@Disabled("Currently disabled due to bug in reset of database") public class Z3SolverEMTest extends SpringTestBase { @BeforeAll diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt index 21f4f8e10a..a395cc8df2 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -39,8 +39,9 @@ class FailModificationEMTest : SpringTestBase(){ val faults = DetectedFaultUtils.getDetectedFaults(solution) - assertEquals(2, faults.size) - assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + val found = faults.filter { it.category == ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION} + + assertEquals(2, found.size) } } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt index 5f2da7aebe..06bfc208db 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt @@ -38,9 +38,9 @@ class FailModificationForbiddenEMTest : SpringTestBase(){ assertTrue(solution.individuals.size >= 1) val faults = DetectedFaultUtils.getDetectedFaults(solution) + .filter { it.category == ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION} assertEquals(1, faults.size) - assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) } } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt index 58b1049626..04377cbbc2 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/ForgottenAuthenticationDisableEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.oracledisable import com.foo.rest.examples.spring.openapi.v3.security.forgottenauthentication.ForgottenAuthenticationController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase @@ -31,14 +32,14 @@ class ForgottenAuthenticationDisableEMTest : SpringTestBase(){ setOption(args, "security", "true") setOption(args, "schemaOracles", "false") setOption(args, "useExperimentalOracles", "true") - setOption(args, "disabledOracleCodes", ExperimentalFaultCategory.IGNORE_ANONYMOUS.code.toString()) + setOption(args, "disabledOracleCodes", DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS.code.toString()) val solution = initAndRun(args) assertTrue(solution.individuals.size >= 1) val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) - assertFalse(ExperimentalFaultCategory.IGNORE_ANONYMOUS in faults) + assertFalse(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS in faults) } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/StackTraceDisableEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/StackTraceDisableEMTest.kt index 49a20a2dc1..5dd276bf1a 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/StackTraceDisableEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/oracledisable/StackTraceDisableEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.oracledisable import com.foo.rest.examples.spring.openapi.v3.security.stacktrace.StackTraceJSONController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase @@ -30,14 +31,14 @@ class StackTraceDisableEMTest : SpringTestBase(){ setOption(args, "security", "true") setOption(args, "schemaOracles", "false") - setOption(args, "disabledOracleCodes", ExperimentalFaultCategory.LEAKED_STACK_TRACES.code.toString()) + setOption(args, "disabledOracleCodes", DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES.code.toString()) val solution = initAndRun(args) assertTrue(solution.individuals.size >= 1) val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) - assertFalse(ExperimentalFaultCategory.LEAKED_STACK_TRACES in faults) + assertFalse(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES in faults) } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/anonymouswrite/SecurityAnonymousWriteEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/anonymouswrite/SecurityAnonymousWriteEMTest.kt index 0d37fcf49f..516d13181c 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/anonymouswrite/SecurityAnonymousWriteEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/anonymouswrite/SecurityAnonymousWriteEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.security.anonymouswrite import com.foo.rest.examples.spring.openapi.v3.security.anonymouswrite.AnonymousWriteController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase @@ -42,11 +43,11 @@ class SecurityAnonymousWriteEMTest : SpringTestBase(){ assertEquals(1, faultsCategories.size) assertEquals(3, faults.size) - assertTrue(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS in faultsCategories) + assertTrue(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS in faultsCategories) // PUT:/api/resources/201/{id} assertTrue(faults.none { - it.category == ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS + it.category == DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS && it.operationId == "PUT:/api/resources/201/{id}" }) } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt index 60fb20e37a..6a66b44727 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/forgottenauthentication/ForgottenAuthenticationEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.security.forgottenauthentication import com.foo.rest.examples.spring.openapi.v3.security.forgottenauthentication.ForgottenAuthenticationController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.rest.data.HttpVerb @@ -44,7 +45,7 @@ class ForgottenAuthenticationEMTest : SpringTestBase(){ val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) assertEquals(1, faults.size) - assertEquals(ExperimentalFaultCategory.IGNORE_ANONYMOUS, faults.first()) + assertEquals(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS, faults.first()) } } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/hiddenaccessible/HiddenAccessibleEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/hiddenaccessible/HiddenAccessibleEMTest.kt index 71c08d1d06..44ebf9c866 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/hiddenaccessible/HiddenAccessibleEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/hiddenaccessible/HiddenAccessibleEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.security.hiddenaccessible import com.foo.rest.examples.spring.openapi.v3.security.hiddenaccessible.HiddenAccessibleController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.rest.data.HttpVerb @@ -46,7 +47,7 @@ class HiddenAccessibleEMTest : SpringTestBase(){ val faults = DetectedFaultUtils.getDetectedFaults(solution) assertTrue(faults.size >= 2) - val hidden = faults.filter{it.category == ExperimentalFaultCategory.HIDDEN_ACCESSIBLE_ENDPOINT} + val hidden = faults.filter{it.category == DefinedFaultCategory.SECURITY_HIDDEN_ACCESSIBLE_ENDPOINT} assertEquals(2, hidden.size) assertNotNull(hidden.find { it.operationId == "GET:/api/resources" }) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceEMTest.kt index a96d0c0cec..244d700325 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.security.stacktrace import com.foo.rest.examples.spring.openapi.v3.security.stacktrace.StackTraceController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.rest.data.HttpVerb @@ -43,11 +44,11 @@ class StackTraceEMTest : SpringTestBase(){ val faultsCategories = DetectedFaultUtils.getDetectedFaultCategories(solution) val faults = DetectedFaultUtils.getDetectedFaults(solution) - assertTrue(ExperimentalFaultCategory.LEAKED_STACK_TRACES in faultsCategories) + assertTrue(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES in faultsCategories) // GET:/api/resources/null-pointer_not_stack_trace assertTrue(faults.none { - it.category == ExperimentalFaultCategory.LEAKED_STACK_TRACES + it.category == DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES && it.operationId == "GET:/api/resources/null-pointer_not_stack_trace" }) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceJSONEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceJSONEMTest.kt index 5af5ee3f07..3f25507481 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceJSONEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/stacktrace/StackTraceJSONEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.openapi.v3.security.stacktrace import com.foo.rest.examples.spring.openapi.v3.security.stacktrace.StackTraceJSONController +import com.webfuzzing.commons.faults.DefinedFaultCategory import org.evomaster.core.problem.enterprise.DetectedFaultUtils import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.problem.rest.data.HttpVerb @@ -41,15 +42,15 @@ class StackTraceJSONEMTest : SpringTestBase(){ val faultsCategories = DetectedFaultUtils.getDetectedFaultCategories(solution) val faults = DetectedFaultUtils.getDetectedFaults(solution) - assertTrue(ExperimentalFaultCategory.LEAKED_STACK_TRACES in faultsCategories) + assertTrue(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES in faultsCategories) assertTrue(faults.any { - it.category == ExperimentalFaultCategory.LEAKED_STACK_TRACES + it.category == DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES && it.operationId == "GET:/api/resources/null-pointer-json" }) assertTrue(faults.any { - it.category == ExperimentalFaultCategory.LEAKED_STACK_TRACES + it.category == DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES && it.operationId == "GET:/api/resources/null-pointer-json-not-list" }) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/html/XSSStoredEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/html/XSSStoredEMTest.kt index 1e4b4fdc00..3991b88714 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/html/XSSStoredEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/html/XSSStoredEMTest.kt @@ -38,7 +38,7 @@ class XSSStoredEMTest : SpringTestBase() { val faults = DetectedFaultUtils.getDetectedFaults(solution) - Assertions.assertTrue(faults.size == 3) + Assertions.assertTrue(faults.size >= 3) val faultCategories = DetectedFaultUtils.getDetectedFaultCategories(solution) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/json/XSSStoredJSONEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/json/XSSStoredJSONEMTest.kt index 1916edbf4d..24f57e188a 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/json/XSSStoredJSONEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/security/xss/stored/json/XSSStoredJSONEMTest.kt @@ -38,7 +38,7 @@ class XSSStoredJSONEMTest : SpringTestBase() { val faults = DetectedFaultUtils.getDetectedFaults(solution) - Assertions.assertTrue(faults.size == 3) + Assertions.assertTrue(faults.size >= 3) val faultCategories = DetectedFaultUtils.getDetectedFaultCategories(solution) diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/db/crossfks/CrossFkEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/db/crossfks/CrossFkEMTest.java index 8ea55d5fcb..e5f367ca90 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/db/crossfks/CrossFkEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/db/crossfks/CrossFkEMTest.java @@ -18,12 +18,12 @@ public static void initClass() throws Exception { SpringTestBase.initClass(new CrossFkController()); } @Test - public void testEnableTaintSampleEM() throws Throwable { + public void testEnableTaintSampleEM_CrossFk() throws Throwable { forceSqlAllColumnInsertion(true); } @Test - public void testDisableTaintSampleEM() throws Throwable { + public void testDisableTaintSampleEM_CrossFk() throws Throwable { forceSqlAllColumnInsertion(false); } diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfilter/EndpointFilterEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfilter/EndpointFilterEMTest.java index 61c1062038..bae1ede6b1 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfilter/EndpointFilterEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfilter/EndpointFilterEMTest.java @@ -20,7 +20,7 @@ public static void initClass() throws Exception { } @Test - public void testRunEM() throws Throwable { + public void testRunEM_EndpointFilter() throws Throwable { runTestHandlingFlakyAndCompilation( "EndpointFilterEM", @@ -28,8 +28,11 @@ public void testRunEM() throws Throwable { 50, (args) -> { - args.add("--endpointTagFilter"); - args.add("Foo"); + setOption(args, "endpointTagFilter", "Foo"); + + setOption(args, "security", "false"); + setOption(args, "schemaOracles", "false"); + setOption(args, "httpOracles", "false"); Solution solution = initAndRun(args); diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java index 6272cf07e7..a691056936 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/endpointfocusandprefix/EndpointFocusAndPrefixTest.java @@ -51,6 +51,8 @@ public void testWithoutFocusOrPrefix() throws Throwable { args.add(baseUrlOfSut); args.add("--bbSwaggerUrl"); args.add(baseUrlOfSut + "/v2/api-docs"); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -122,6 +124,8 @@ public void testRunBlackboxWithFocusWithoutParameters() throws Throwable { args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointFocus"); args.add(endpointFocus); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -164,6 +168,8 @@ public void testRunBlackboxWithFocusWithParameters() throws Throwable { args.add("--endpointFocus"); args.add(endpointFocus); setOption(args, "advancedBlackBoxCoverage", "false"); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -204,6 +210,8 @@ public void testRunBlackboxWithFocusOneEndpoint() throws Throwable { args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointFocus"); args.add(endpointFocus); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -244,6 +252,8 @@ public void testRunBlackboxWithPrefixWithoutParameters() throws Throwable { args.add(baseUrlOfSut + "/v2/api-docs"); args.add("--endpointPrefix"); args.add(endpointPrefix); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -285,6 +295,8 @@ public void testRunBlackboxWithPrefixWithParameters() throws Throwable { args.add("--endpointPrefix"); args.add(endpointPrefix); setOption(args, "advancedBlackBoxCoverage", "false"); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); @@ -326,6 +338,8 @@ public void testRunBlackboxPrefixNonExistingFocusValidPrefix() throws Throwable args.add("--endpointPrefix"); args.add(endpointPrefix); setOption(args, "advancedBlackBoxCoverage", "false"); + setOption(args, "security", "false"); + setOption(args, "httpOracles", "false"); // no endpointFocus or endpointPrefix is provided Solution solution = initAndRun(args); diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index bdd5060bd3..acc6a89a58 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2872,27 +2872,22 @@ class EMConfig { var security = true - @Experimental @Cfg("To apply SSRF detection as part of security testing.") @DependsOnTrueFor("security") - var ssrf = false + var ssrf = true - @Experimental @Cfg("To apply XSS detection as part of security testing.") @DependsOnTrueFor("security") - var xss = false + var xss = true - @Experimental @Cfg("To apply SQLi detection as part of security testing.") @DependsOnTrueFor("security") - var sqli = false + var sqli = true - @Experimental @Cfg("Injected sleep duration (in seconds) used inside the malicious payload to detect time-based vulnerabilities.") @DependsOnTrueFor("sqli") var sqliInjectedSleepDurationMs = 5000 - @Experimental @Cfg("Maximum allowed baseline response time (in milliseconds) before the malicious payload is applied.") @DependsOnTrueFor("sqli") var sqliBaselineMaxResponseTimeMs = 2000 diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 32381812d7..e901815db0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -15,15 +15,6 @@ enum class ExperimentalFaultCategory( //Likely this one is not really viable //SECURITY_ALLOW_MODIFICATION_BY_ALL(985, "Resource Created By An User Can Be Modified By All Other Users", "createdResourceCanBeModifiedByEveryone", // "TODO") - IGNORE_ANONYMOUS(900, "A Protected Resource Is Accessible Without Providing Any Authentication", - "ignoreAnonymous", - "TODO"), - ANONYMOUS_MODIFICATIONS(901, "Anonymous Modifications", - "anonymousModifications", - "TODO"), - LEAKED_STACK_TRACES(902, "Leaked Stack Trace", - "leakedStackTrace", - "TODO"), HIDDEN_ACCESSIBLE_ENDPOINT(903, "Hidden Accessible Endpoint", "hiddenAccessible", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestCallAction.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestCallAction.kt index f00745237a..90a9d41a18 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestCallAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestCallAction.kt @@ -114,18 +114,8 @@ class RestCallAction( if(!isPotentialActionForCreation()){ throw IllegalStateException("Location Ids are meaningful only for POST operations") } - //return path.lastElement() - /* - previous was problematic, as ids were not unique. it wasn't an issue for chains, but it - became major issue for cleanups. - but, using local ids has its own issues (only defined once mounted into an individual). - TODO will need to check for side-effects, might require some more refactoring - */ - if(weakReference != null){ - throw IllegalStateException("'weakReference' has not been handled yet ") - } if(!hasLocalId()){ - throw IllegalStateException("Location ID must be present when computing a creationLocationId") + throw IllegalStateException("Local ID must be present when computing a creationLocationId") } val k = getLocalId() // TODO could skip k if non-ambiguous. otherwise, counter could start from 0 (ie need a map for k values) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/RestSecurityOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/RestSecurityOracle.kt index 40d73ea2a9..97616f397c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/RestSecurityOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/RestSecurityOracle.kt @@ -327,7 +327,7 @@ class RestSecurityOracle { actionResults: List, fv: FitnessValue ) { - if (!config.isEnabledFaultCategory(ExperimentalFaultCategory.LEAKED_STACK_TRACES)) { + if (!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES)) { return } @@ -339,10 +339,10 @@ class RestSecurityOracle { if(r.getStatusCode() == 500 && r.getBody() != null && StackTraceUtils.looksLikeStackTrace(r.getBody()!!)){ val scenarioId = idMapper.handleLocalTarget( - idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.LEAKED_STACK_TRACES, a.getName()) + idMapper.getFaultDescriptiveId(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES, a.getName()) ) fv.updateTarget(scenarioId, 1.0, index) - r.addFault(DetectedFault(ExperimentalFaultCategory.LEAKED_STACK_TRACES, a.getName(), null)) + r.addFault(DetectedFault(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES, a.getName(), null)) } } } @@ -352,7 +352,7 @@ class RestSecurityOracle { actionResults: List, fv: FitnessValue ){ - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HIDDEN_ACCESSIBLE_ENDPOINT)){ + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_HIDDEN_ACCESSIBLE_ENDPOINT)){ return } @@ -385,10 +385,10 @@ class RestSecurityOracle { // we also consider 403, in case API just give it by default for security reasons val scenarioId = idMapper.handleLocalTarget( - idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.HIDDEN_ACCESSIBLE_ENDPOINT, target.getName()) + idMapper.getFaultDescriptiveId(DefinedFaultCategory.SECURITY_HIDDEN_ACCESSIBLE_ENDPOINT, target.getName()) ) fv.updateTarget(scenarioId, 1.0, index+1) - data.addFault(DetectedFault(ExperimentalFaultCategory.HIDDEN_ACCESSIBLE_ENDPOINT, target.getName(), null)) + data.addFault(DetectedFault(DefinedFaultCategory.SECURITY_HIDDEN_ACCESSIBLE_ENDPOINT, target.getName(), null)) } } } @@ -439,7 +439,7 @@ class RestSecurityOracle { actionResults: List, fv: FitnessValue ) { - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS)){ + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS)){ return } @@ -470,10 +470,10 @@ class RestSecurityOracle { } val scenarioId = idMapper.handleLocalTarget( - idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS, a.getName()) + idMapper.getFaultDescriptiveId(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS, a.getName()) ) fv.updateTarget(scenarioId, 1.0, index) - r.addFault(DetectedFault(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS, a.getName(), null)) + r.addFault(DetectedFault(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS, a.getName(), null)) } } } @@ -484,7 +484,7 @@ class RestSecurityOracle { fv: FitnessValue ) { - if (!config.isEnabledFaultCategory(ExperimentalFaultCategory.IGNORE_ANONYMOUS)) { + if (!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS)) { return } @@ -504,10 +504,10 @@ class RestSecurityOracle { if(a.auth is NoAuth && faultyEndpoints.contains(a.getName()) && StatusGroup.G_2xx.isInGroup(r.getStatusCode())){ val scenarioId = idMapper.handleLocalTarget( - idMapper.getFaultDescriptiveId(ExperimentalFaultCategory.IGNORE_ANONYMOUS, a.getName()) + idMapper.getFaultDescriptiveId(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS, a.getName()) ) fv.updateTarget(scenarioId, 1.0, index) - r.addFault(DetectedFault(ExperimentalFaultCategory.IGNORE_ANONYMOUS, a.getName(), null)) + r.addFault(DetectedFault(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS, a.getName(), null)) } } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestSecurityBuilder.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestSecurityBuilder.kt index f6a5e0e748..e3652b2015 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestSecurityBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestSecurityBuilder.kt @@ -331,14 +331,14 @@ class RestSecurityBuilder : TimeBoxedPhase { handleExistenceLeakage() } - if (!config.isEnabledFaultCategory(ExperimentalFaultCategory.LEAKED_STACK_TRACES)) { + if (!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_LEAKED_STACK_TRACES)) { log.debug("Skipping experimental security test for stack traces as disabled in configuration") } else { if(hasPhaseTimedOut()) return handleStackTraceCheck() } - if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HIDDEN_ACCESSIBLE_ENDPOINT)){ + if(config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_HIDDEN_ACCESSIBLE_ENDPOINT)){ if(hasPhaseTimedOut()) return handleHiddenAccessibleEndpoint() } @@ -368,14 +368,14 @@ class RestSecurityBuilder : TimeBoxedPhase { handleNotRecognizedAuthenticated() } - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.IGNORE_ANONYMOUS)) { + if(!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_IGNORE_ANONYMOUS)) { log.debug("Skipping experimental security test for forgotten authentication as disabled in configuration") } else { if(hasPhaseTimedOut()) return handleForgottenAuthentication() } - if (!config.isEnabledFaultCategory(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS)) { + if (!config.isEnabledFaultCategory(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS)) { log.debug("Skipping experimental security test for anonymous write as disabled in configuration") } else { if(hasPhaseTimedOut()) return @@ -734,7 +734,7 @@ class RestSecurityBuilder : TimeBoxedPhase { val faultsCategories = DetectedFaultUtils.getDetectedFaultCategories(evaluatedIndividual) - if(ExperimentalFaultCategory.ANONYMOUS_MODIFICATIONS in faultsCategories){ + if(DefinedFaultCategory.SECURITY_ANONYMOUS_MODIFICATIONS in faultsCategories){ val added = archive.addIfNeeded(evaluatedIndividual) assert(added) continue@mainloop @@ -1370,7 +1370,6 @@ class RestSecurityBuilder : TimeBoxedPhase { ) val getAction = second.seeMainExecutableActions().last() as RestCallAction - getAction.resetLocalIdRecursively() val genes = GeneUtils.getAllStringFields(getAction.parameters) .filter { it.staticCheckIfImpactPhenotype() } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt index 086fff0b55..8bd45e1144 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt @@ -1104,6 +1104,11 @@ class StringGene( override fun unsafeSetFromStringValue(value: String): Boolean { + /* + TODO most likely this will need to be refactored, to require either "" or '' quotes. + See NullableGene + */ + this.value = value selectedSpecialization = -1 diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/NullableGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/NullableGene.kt index 3a9db92713..ec69bcc6ea 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/NullableGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/NullableGene.kt @@ -127,4 +127,12 @@ class NullableGene(name: String, && this.nullLabel == other.nullLabel } + override fun unsafeSetFromStringValue(value: String): Boolean { + if(value == nullLabel){ + isActive = false + return true + } else { + return gene.unsafeSetFromStringValue(value) + } + } } \ No newline at end of file diff --git a/docs/options.md b/docs/options.md index a9dcaa0c23..0608665d82 100644 --- a/docs/options.md +++ b/docs/options.md @@ -213,6 +213,10 @@ There are 3 types of options: |`snapshotInterval`| __Double__. If positive, check how often, in percentage % of the budget, to collect statistics snapshots. For example, every 5% of the time. *Constraints*: `max=50.0`. *Default value*: `-1.0`.| |`snapshotStatisticsFile`| __String__. Where the snapshot file (if any) is going to be written (in CSV format). *Default value*: `snapshot.csv`.| |`specializeSQLGeneSelection`| __Boolean__. Whether to specialize sql gene selection to mutation. *Default value*: `true`.| +|`sqli`| __Boolean__. To apply SQLi detection as part of security testing. *Depends on*: `security=true`. *Default value*: `true`.| +|`sqliBaselineMaxResponseTimeMs`| __Int__. Maximum allowed baseline response time (in milliseconds) before the malicious payload is applied. *Depends on*: `sqli=true`. *Default value*: `2000`.| +|`sqliInjectedSleepDurationMs`| __Int__. Injected sleep duration (in seconds) used inside the malicious payload to detect time-based vulnerabilities. *Depends on*: `sqli=true`. *Default value*: `5000`.| +|`ssrf`| __Boolean__. To apply SSRF detection as part of security testing. *Depends on*: `security=true`. *Default value*: `true`.| |`startNumberOfMutations`| __Int__. Number of applied mutations on sampled individuals, at the start of the search. *Constraints*: `min=0.0`. *Default value*: `1`.| |`startingPerOfGenesToMutate`| __Double__. Specify a starting percentage of genes of an individual to mutate. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.| |`statisticsColumnId`| __String__. An id that will be part as a column of the statistics file (if any is generated). *Default value*: `-`.| @@ -242,6 +246,7 @@ There are 3 types of options: |`writeWFCReport`| __Boolean__. Output a JSON file representing statistics of the fuzzing session, written in the WFC Report format. This also includes a index.html web application to visualize such data. *Depends on*: `createTests=true`. *Default value*: `true`.| |`writeWFCReportExcludeWebApp`| __Boolean__. If creating a WFC Report as output, specify if should not generate the index.html web app, i.e., only the JSON report file will be created. *Default value*: `false`.| |`xoverProbability`| __Double__. Probability of applying crossover operation (if any is used in the search algorithm). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.7`.| +|`xss`| __Boolean__. To apply XSS detection as part of security testing. *Depends on*: `security=true`. *Default value*: `true`.| ## Experimental Command-Line Options @@ -324,10 +329,6 @@ There are 3 types of options: |`seedTestCasesPath`| __String__. File path where the seeded test cases are located. *Default value*: `postman.postman_collection.json`.| |`skipAIModelUpdateWhenResponseIs5xx`| __Boolean__. Determines whether the AI response classifier skips model updates when the response indicates a server-side error with status code 5xx. *Default value*: `false`.| |`skipAIModelUpdateWhenResponseIsNot2xxOr400`| __Boolean__. Determines whether the AI response classifier skips model updates when the response is not 2xx or 400. *Default value*: `false`.| -|`sqli`| __Boolean__. To apply SQLi detection as part of security testing. *Depends on*: `security=true`. *Default value*: `false`.| -|`sqliBaselineMaxResponseTimeMs`| __Int__. Maximum allowed baseline response time (in milliseconds) before the malicious payload is applied. *Depends on*: `sqli=true`. *Default value*: `2000`.| -|`sqliInjectedSleepDurationMs`| __Int__. Injected sleep duration (in seconds) used inside the malicious payload to detect time-based vulnerabilities. *Depends on*: `sqli=true`. *Default value*: `5000`.| -|`ssrf`| __Boolean__. To apply SSRF detection as part of security testing. *Depends on*: `security=true`. *Default value*: `false`.| |`structureMutationProFS`| __Double__. Specify a probability of applying structure mutator during the focused search. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`structureMutationProbStrategy`| __Enum__. Specify a strategy to handle a probability of applying structure mutator during the focused search. *Valid values*: `SPECIFIED, SPECIFIED_FS, DPC_TO_SPECIFIED_BEFORE_FS, DPC_TO_SPECIFIED_AFTER_FS, ADAPTIVE_WITH_IMPACT`. *Default value*: `SPECIFIED`.| |`sutDistEnvVarName`| __String__. Specify name of the environment variable that provides the the base distribution directory of the SUT, e.g., 'dist' directory of WFD. *Default value*: `""`.| @@ -344,4 +345,3 @@ There are 3 types of options: |`vulnerableInputClassificationStrategy`| __Enum__. Strategy to classify inputs for potential vulnerability classes related to an REST endpoint. *Valid values*: `MANUAL, LLM`. *Default value*: `MANUAL`.| |`wbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for white-box (wb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.| |`writeSnapshotTestsIntervalInSeconds`| __Int__. The size (in seconds) of the interval that the snapshots will be printed, if enabled. *Default value*: `3600`.| -|`xss`| __Boolean__. To apply XSS detection as part of security testing. *Depends on*: `security=true`. *Default value*: `false`.| diff --git a/pom.xml b/pom.xml index a4e2323fac..325e1db70e 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 17 2.2.20 true - 0.2.0 + 0.4.0 5.14.2 1.14.2 3.1.5 diff --git a/release_notes.md b/release_notes.md index 244992872d..d4c8dda2eb 100644 --- a/release_notes.md +++ b/release_notes.md @@ -5,6 +5,7 @@ Under development in `master` branch. ### New Features - Each new "phase" (e.g., minimization and security testing) after the search process has now a time-budget which is proportional to the search budget (default 10 percent). +- Upgraded to WFC 0.4.0, which includes improvements to the Web Report (e.g., "low-code" view), and generation of self-contained index.html file (for whom cannot run a Python HTTP server on their machines, e.g., managers in industry with limited permissions on their corporate laptop). ### Miscellaneous