Merge remote-tracking branch 'origin/master' into sql-multi-column-foreign-keys

Author: jgaleotti

core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/IntegrationTestRestBase.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler
 import org.evomaster.core.problem.rest.service.RestSecurityBuilder
 import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.service.Archive
+import org.evomaster.core.search.service.time.ExecutionPhaseController
 import org.evomaster.core.seeding.service.rest.PirToRest
 import org.evomaster.e2etests.utils.RestTestBase
 import org.junit.jupiter.api.BeforeEach
@@ -60,6 +61,8 @@ abstract class IntegrationTestRestBase : RestTestBase() {
 
     fun getSecurityOracle() = injector.getInstance(RestSecurityOracle::class.java)
 
+    fun getExecutionPhaseController() = injector.getInstance(ExecutionPhaseController::class.java)
+
     /**
      * Create and evaluate an individual
      */

core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/securityrestoracle/SecurityForbiddenOperationTest.kt

@@ -37,6 +37,12 @@ class SecurityForbiddenOperationTest : IntegrationTestRestBase() {
         SecurityForbiddenOperationApplication.reset()
     }
 
+    private fun makeSureCanRunSecurityPhase(){
+        val epc = getExecutionPhaseController()
+        epc.markStartingSearch()
+        epc.markStartingSecurity()
+    }
+
     @Test
     fun testDeletePatch(){
 
@@ -107,6 +113,7 @@ class SecurityForbiddenOperationTest : IntegrationTestRestBase() {
         val security = getSecurityRest()
         val config = getEMConfig()
 
+
         config.security = true
         config.schemaOracles = false
 
@@ -129,6 +136,7 @@ class SecurityForbiddenOperationTest : IntegrationTestRestBase() {
         val added = archive.addIfNeeded(ind)
         assertTrue(added)
 
+        makeSureCanRunSecurityPhase()
         val solution = security.applySecurityPhase()
 
         val target = solution.individuals.find { it.hasAnyPotentialFault() }!!
@@ -186,7 +194,7 @@ class SecurityForbiddenOperationTest : IntegrationTestRestBase() {
         val otherAdded = archive.addIfNeeded(other)
         assertTrue(otherAdded)
 
-
+        makeSureCanRunSecurityPhase()
         val solution = security.applySecurityPhase()
 
         val target = solution.individuals.find { it.hasAnyPotentialFault() }!!
@@ -233,6 +241,7 @@ class SecurityForbiddenOperationTest : IntegrationTestRestBase() {
         val added = archive.addIfNeeded(ind)
         assertTrue(added)
 
+        makeSureCanRunSecurityPhase()
         val solution = security.applySecurityPhase()
 
         val target = solution.individuals.find { it.hasAnyPotentialFault() }!!

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -253,23 +253,28 @@ class Main {
             val idMapper = injector.getInstance(IdMapper::class.java)
             val epc = injector.getInstance(ExecutionPhaseController::class.java)
 
-            var solution = run(injector, controllerInfo)
+            run(injector, controllerInfo)
 
             //save data regarding the search phase
             writeOverallProcessData(injector)
             writeDependencies(injector)
-            writeImpacts(injector, solution)
+            writeImpacts(injector)
             writeExecuteInfo(injector)
-
             logTimeSearchInfo(injector, config)
 
+
+            //-----------------------------------------------------------
             //apply new phases
+            phaseMinimizer(injector, config, epc)
             // 403 are usually not found during search, but created on purpose during security phase by
             // mixing different users in the same test. as some http oracles might depend on 403s,
             // we make sure to run security phase first
-            solution = phaseSecurity(injector, config, epc, solution)
-            solution = phaseHttpOracle(injector, config, epc, solution)
-            solution = phaseFlaky(injector, config, epc, solution)
+            phaseSecurity(injector, config, epc)
+            phaseHttpOracle(injector, config, epc)
+            phaseFlaky(injector, config, epc)
+            //-----------------------------------------------------------
+
+            val solution = extractSolution(injector)
 
             epc.markStartingWriteOutput()
             val suites = writeTests(injector, solution, controllerInfo)
@@ -278,7 +283,6 @@ class Main {
             writeCoveredTargets(injector, solution)
             //NOTE: the WRITE_OUTPUT phase here would be not computed, as it is not finished yet...
             writeStatistics(injector, solution)
-            //FIXME if other phases after search, might get skewed data on 100% snapshots...
 
             resetExternalServiceHandler(injector)
             // Stop the WM before test execution
@@ -412,17 +416,35 @@ class Main {
             }
         }
 
+        private fun extractSolution(injector: Injector): Solution<*> {
+            val archive = injector.getInstance(Archive::class.java)
+            return archive.extractSolution()
+        }
+
+        private fun phaseMinimizer(
+            injector: Injector,
+            config: EMConfig,
+            epc: ExecutionPhaseController,
+        ){
+            if(!config.minimize){
+                return
+            }
+            epc.markStartingMinimization()
+
+            val minimizer = injector.getInstance(Key.get(object : TypeLiteral<Minimizer<*>>() {}))
+            minimizer.applyPhase()
+        }
+
         private fun phaseFlaky(
             injector: Injector,
             config: EMConfig,
             epc: ExecutionPhaseController,
-            solution: Solution<*>
-        ): Solution<*> {
+        ) {
             if (!config.handleFlakiness){
-                return solution
+                return
             }
 
-            return when (config.problemType) {
+            when (config.problemType) {
                 EMConfig.ProblemType.REST -> {
                     LoggingUtil.getInfoLogger().info("Starting to apply flaky detection")
                     epc.markStartingFlakiness()
@@ -432,31 +454,25 @@ class Main {
                 } else -> {
                     LoggingUtil.getInfoLogger()
                         .warn("Flakiness detection phase currently not handled for problem type: ${config.problemType}")
-                    solution
                 }
             }
-
-
-
         }
 
 
         private fun phaseSecurity(
             injector: Injector,
             config: EMConfig,
             epc: ExecutionPhaseController,
-            solution: Solution<*>
-        ): Solution<*> {
+        ) {
             if (!config.security) {
-                return solution
+                return
             }
             //apply security testing phase
-            LoggingUtil.getInfoLogger().info("Starting to apply security testing")
             epc.markStartingSecurity()
 
             //TODO might need to reset stc, and print some updated info again
 
-            return when (config.problemType) {
+            when (config.problemType) {
                 EMConfig.ProblemType.REST -> {
                     val securityRest = injector.getInstance(RestSecurityBuilder::class.java)
                     securityRest.applySecurityPhase()
@@ -465,26 +481,22 @@ class Main {
                 else -> {
                     LoggingUtil.getInfoLogger()
                         .warn("Security phase currently not handled for problem type: ${config.problemType}")
-                    solution
                 }
             }
         }
 
         private fun phaseHttpOracle(
             injector: Injector,
             config: EMConfig,
-            epc: ExecutionPhaseController,
-            solution: Solution<*>
-        ): Solution<*> {
+            epc: ExecutionPhaseController
+        ) {
 
-            return if (config.httpOracles && config.problemType == EMConfig.ProblemType.REST) {
-                LoggingUtil.getInfoLogger().info("Starting to apply HTTP")
+            if (config.httpOracles && config.problemType == EMConfig.ProblemType.REST) {
+                LoggingUtil.getInfoLogger().info("Starting to apply HTTP oracle detection")
                 epc.markStartingAdditionalOracles()
 
                 val httpSemanticsService = injector.getInstance(HttpSemanticsService::class.java)
                 httpSemanticsService.applyHttpSemanticsPhase()
-            } else {
-                solution
             }
         }
 
@@ -1056,14 +1068,16 @@ class Main {
          * save derived impacts of genes of actions.
          * info is designed for experiment analysis
          */
-        private fun writeImpacts(injector: Injector, solution: Solution<*>) {
+        private fun writeImpacts(injector: Injector) {
 
             val config = injector.getInstance(EMConfig::class.java)
 
             if (!config.exportImpacts) {
                 return
             }
 
+            val solution = extractSolution(injector)
+
             val am = injector.getInstance(ArchiveImpactSelector::class.java)
             am.exportImpacts(solution)
         }

core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt

@@ -48,11 +48,16 @@ class GraphQLBlackBoxModule(
         bind(object : TypeLiteral<Archive<*>>() {})
                 .to(object : TypeLiteral<Archive<GraphQLIndividual>>() {})
 
+        bind(Archive::class.java)
+            .to(object : TypeLiteral<Archive<GraphQLIndividual>>() {})
+
         bind(object : TypeLiteral<Minimizer<GraphQLIndividual>>(){})
                 .asEagerSingleton()
 
         bind(object : TypeLiteral<Minimizer<*>>(){})
-                .asEagerSingleton()
+            .to(object : TypeLiteral<Minimizer<GraphQLIndividual>>() {})
+            .asEagerSingleton()
+
 
         bind(object : TypeLiteral<FlakinessDetector<GraphQLIndividual>>(){})
             .asEagerSingleton()

core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt

@@ -51,10 +51,14 @@ class GraphQLModule : EnterpriseModule() {
         bind(object : TypeLiteral<Archive<*>>() {})
                 .to(object : TypeLiteral<Archive<GraphQLIndividual>>() {})
 
+        bind(Archive::class.java)
+            .to(object : TypeLiteral<Archive<GraphQLIndividual>>() {})
+
         bind(object : TypeLiteral<Minimizer<GraphQLIndividual>>(){})
                 .asEagerSingleton()
 
         bind(object : TypeLiteral<Minimizer<*>>(){})
+                .to(object : TypeLiteral<Minimizer<GraphQLIndividual>>(){})
                 .asEagerSingleton()
 
         bind(object : TypeLiteral<FlakinessDetector<GraphQLIndividual>>(){})

core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt

@@ -18,14 +18,16 @@ import org.evomaster.core.search.Solution
 import org.evomaster.core.search.service.Archive
 import org.evomaster.core.search.service.IdMapper
 import org.evomaster.core.search.service.Randomness
+import org.evomaster.core.search.service.time.ExecutionPhaseController
+import org.evomaster.core.search.service.time.TimeBoxedPhase
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import javax.annotation.PostConstruct
 
 /**
  * Service class used to verify HTTP semantics properties.
  */
-class HttpSemanticsService {
+class HttpSemanticsService : TimeBoxedPhase{
 
     companion object {
         private val log: Logger = LoggerFactory.getLogger(HttpSemanticsService::class.java)
@@ -52,6 +54,9 @@ class HttpSemanticsService {
     @Inject
     private lateinit var builder: RestIndividualBuilder
 
+    @Inject
+    private lateinit var epc: ExecutionPhaseController
+
     /**
      * All actions that can be defined from the OpenAPI schema
      */
@@ -72,7 +77,13 @@ class HttpSemanticsService {
 
     }
 
+    override fun applyPhase() {
+        applyHttpSemanticsPhase()
+    }
 
+    override fun hasPhaseTimedOut(): Boolean {
+        return epc.hasPhaseTimedOut(ExecutionPhaseController.Phase.ADDITIONAL_ORACLES)
+    }
 
     fun applyHttpSemanticsPhase(): Solution<RestIndividual>{
 
@@ -92,12 +103,15 @@ class HttpSemanticsService {
 //        – JSON-Merge-Patch: partial update should not impact other fields. Can have GET, PATCH, and GET to verify it
 
 
+        if(hasPhaseTimedOut()) return
         // – 2xx GET on K : follow by success 2xx DELETE, should then give 404 on GET k (adding up to 2 calls)
         deleteShouldDelete()
 
+        if(hasPhaseTimedOut()) return
         // –  A repeated followup PUT with 201 on same endpoint should not return 201 (must enforce 200 or 204)
         putRepeatedCreated()
 
+        if(hasPhaseTimedOut()) return
         sideEffectsOfFailedModification()
     }
 
@@ -114,6 +128,8 @@ class HttpSemanticsService {
 
         putOperations.forEach { put ->
 
+            if(hasPhaseTimedOut()) return
+
             val creates = RestIndividualSelectorUtils.findAndSlice(
                 individualsInSolution,
                 HttpVerb.PUT,
@@ -160,6 +176,8 @@ class HttpSemanticsService {
 
         deleteOperations.forEach { del ->
 
+            if(hasPhaseTimedOut()) return
+
             val successDelete = RestIndividualSelectorUtils.findAndSlice(
                 individualsInSolution,
                 HttpVerb.DELETE,
@@ -227,6 +245,8 @@ class HttpSemanticsService {
 
             modifyOperations.forEach { modOp ->
 
+                if(hasPhaseTimedOut()) return
+
                 val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == modOp.path }
                     ?: return@forEach