Update EmTest

MohsenTaheriShalmani · MohsenTaheriShalmani · commit bb069b4520f3 · 2026-03-07T15:32:31.000+01:00
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/AIClassificationEMTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/AIClassificationEMTestBase.kt
@@ -39,31 +39,53 @@ abstract class AIClassificationEMTestBase : SpringTestBase(){
         return ei
     }
 
+    private fun isWeakClassifier(
+        model: AIResponseClassifier,
+        action: RestCallAction,
+        weaknessThreshold: Double
+    ): Boolean {
+
+        val metrics = model.estimateMetrics(action.endpoint)
+
+        return metrics.precision400 <= weaknessThreshold
+                || metrics.sensitivity400 <= weaknessThreshold
+                || metrics.specificity <= weaknessThreshold
+                || metrics.npv <= weaknessThreshold
+    }
+
     protected fun verifyModel(
         injector: Injector,
         ok2xx: List<RestCallAction>,
         fail400: List<RestCallAction>,
-        threshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold
+        repairThreshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold,
+        weaknessThreshold: Double = injector.getInstance(EMConfig::class.java).aIResponseClassifierWeaknessThreshold
     ) {
 
         val model = injector.getInstance(AIResponseClassifier::class.java)
         model.disableLearning() // no side-effects
 
+
         for(ok in ok2xx){
+
+            if (isWeakClassifier(model, ok, weaknessThreshold)) continue
+
             val resOK = evaluateAction(injector, ok)
             assertTrue(resOK.getStatusCode() in 200..299)
             val mOK= model.classify(ok)
             assertTrue(
-                mOK.probabilityOf400() < threshold,
+                mOK.probabilityOf400() < repairThreshold,
                 "Too high probability of 400 for OK ${ok.getName()}: ${mOK.probabilityOf400()}")
         }
 
         for(fail in fail400) {
+
+            if (isWeakClassifier(model, fail, weaknessThreshold)) continue
+
             val resFail = evaluateAction(injector, fail)
             assertEquals(400, resFail.getStatusCode())
             val mFail = model.classify(fail)
             assertTrue(
-                mFail.probabilityOf400() >= threshold,
+                mFail.probabilityOf400() >= repairThreshold,
                 "Too low probability of 400 for Fail ${fail.getName()}: ${mFail.probabilityOf400()}"
             )
         }
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/allornone/ACAllOrNoneEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/allornone/ACAllOrNoneEMTest.kt
@@ -27,7 +27,7 @@ class ACAllOrNoneEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/arithmetic/ACArithmeticEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/arithmetic/ACArithmeticEMTest.kt
@@ -27,7 +27,7 @@ class ACArithmeticEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/basic/ACBasicEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/basic/ACBasicEMTest.kt
@@ -26,7 +26,7 @@ class ACBasicEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/imply/ACImplyEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/imply/ACImplyEMTest.kt
@@ -35,7 +35,7 @@ class ACImplyEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/mixed/ACMixedEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/mixed/ACMixedEMTest.kt
@@ -36,7 +36,7 @@ class ACMixedEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/onlyone/ACOnlyOneEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/onlyone/ACOnlyOneEMTest.kt
@@ -33,7 +33,7 @@ class ACOnlyOneEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/or/ACOrEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/or/ACOrEMTest.kt
@@ -32,7 +32,7 @@ class ACOrEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/required/ACRequiredEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/required/ACRequiredEMTest.kt
@@ -31,7 +31,7 @@ class ACRequiredEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/zeroorone/ACZeroOrOneEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/zeroorone/ACZeroOrOneEMTest.kt
@@ -35,7 +35,7 @@ class ACZeroOrOneEMTest : AIClassificationEMTestBase() {
         testRunEM(AIResponseClassifierModel.DETERMINISTIC)
     }
 
-    @Disabled
+
     @Test
     fun testRunGaussian(){
         testRunEM(AIResponseClassifierModel.GAUSSIAN)
diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt
@@ -1563,7 +1563,7 @@ class EMConfig {
     @PercentageAsProbability(false)
     @Cfg("If using THRESHOLD for AI Classification Repair, specify its value." +
             " All classifications with probability equal or above such threshold value will be accepted.")
-    var classificationRepairThreshold = 0.8
+    var classificationRepairThreshold = 0.5
 
     @Experimental
     @Cfg("Specify how the classification of actions's response will be used to execute a possible repair on the action.")
@@ -1602,7 +1602,7 @@ class EMConfig {
     @Experimental
     @Cfg("Minimum confidence threshold required for the AI response classifier to decide" +
             "whether to send a request as-is or attempt a repair.")
-    var aIResponseClassifierWeaknessThreshold = 0.4
+    var aIResponseClassifierWeaknessThreshold = 0.8
 
     @Cfg("Output a JSON file representing statistics of the fuzzing session, written in the WFC Report format." +
             " This also includes a index.html web application to visualize such data.")
diff --git a/docs/options.md b/docs/options.md
@@ -245,7 +245,7 @@ There are 3 types of options:
 |Options|Description|
 |---|---|
 |`aIClassificationMetrics`| __Enum__. Determines which metric-tracking strategy is used by the AI response classifier. *Valid values*: `TIME_WINDOW, FULL_HISTORY`. *Default value*: `FULL_HISTORY`.|
-|`aIResponseClassifierWeaknessThreshold`| __Double__. Minimum confidence threshold required for the AI response classifier to decidewhether to send a request as-is or attempt a repair. *Default value*: `0.4`.|
+|`aIResponseClassifierWeaknessThreshold`| __Double__. Minimum confidence threshold required for the AI response classifier to decidewhether to send a request as-is or attempt a repair. *Default value*: `0.8`.|
 |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.|
 |`aiClassifierRepairActivation`| __Enum__. Specify how the classification of actions's response will be used to execute a possible repair on the action. *Valid values*: `PROBABILITY, THRESHOLD`. *Default value*: `THRESHOLD`.|
 |`aiEncoderType`| __Enum__. The encoding strategy applied to transform raw data to the encoded version. *Valid values*: `RAW, NORMAL, UNIT_NORMAL`. *Default value*: `RAW`.|
@@ -259,7 +259,7 @@ There are 3 types of options:
 |`breederTruncationFraction`| __Double__. Breeder GA: fraction of top individuals to keep in parents pool (truncation). *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.|
 |`cgaNeighborhoodModel`| __Enum__. Cellular GA: neighborhood model (RING, L5, C9, C13). *Valid values*: `RING, L5, C9, C13`. *Default value*: `RING`.|
-|`classificationRepairThreshold`| __Double__. If using THRESHOLD for AI Classification Repair, specify its value. All classifications with probability equal or above such threshold value will be accepted. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.|
+|`classificationRepairThreshold`| __Double__. If using THRESHOLD for AI Classification Repair, specify its value. All classifications with probability equal or above such threshold value will be accepted. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.5`.|
 |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.|
 |`dockerLocalhost`| __Boolean__. Replace references to 'localhost' to point to the actual host machine. Only needed when running EvoMaster inside Docker. *Default value*: `false`.|
 |`dpcTargetTestSize`| __Int__. Specify a max size of a test to be targeted when either DPC_INCREASING or DPC_DECREASING is enabled. *Default value*: `1`.|

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ class ACAllOrNoneEMTest : AIClassificationEMTestBase() {`
`27`	`27`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`28`	`28`	`}`
`29`	`29`
`30`		`- @Disabled`
	`30`	`+`
`31`	`31`	`@Test`
`32`	`32`	`fun testRunGaussian(){`
`33`	`33`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ class ACArithmeticEMTest : AIClassificationEMTestBase() {`
`27`	`27`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`28`	`28`	`}`
`29`	`29`
`30`		`- @Disabled`
	`30`	`+`
`31`	`31`	`@Test`
`32`	`32`	`fun testRunGaussian(){`
`33`	`33`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ class ACBasicEMTest : AIClassificationEMTestBase() {`
`26`	`26`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`27`	`27`	`}`
`28`	`28`
`29`		`- @Disabled`
	`29`	`+`
`30`	`30`	`@Test`
`31`	`31`	`fun testRunGaussian(){`
`32`	`32`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ class ACImplyEMTest : AIClassificationEMTestBase() {`
`35`	`35`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`36`	`36`	`}`
`37`	`37`
`38`		`- @Disabled`
	`38`	`+`
`39`	`39`	`@Test`
`40`	`40`	`fun testRunGaussian(){`
`41`	`41`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ class ACMixedEMTest : AIClassificationEMTestBase() {`
`36`	`36`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`37`	`37`	`}`
`38`	`38`
`39`		`- @Disabled`
	`39`	`+`
`40`	`40`	`@Test`
`41`	`41`	`fun testRunGaussian(){`
`42`	`42`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ class ACOnlyOneEMTest : AIClassificationEMTestBase() {`
`33`	`33`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`34`	`34`	`}`
`35`	`35`
`36`		`- @Disabled`
	`36`	`+`
`37`	`37`	`@Test`
`38`	`38`	`fun testRunGaussian(){`
`39`	`39`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ class ACOrEMTest : AIClassificationEMTestBase() {`
`32`	`32`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`33`	`33`	`}`
`34`	`34`
`35`		`- @Disabled`
	`35`	`+`
`36`	`36`	`@Test`
`37`	`37`	`fun testRunGaussian(){`
`38`	`38`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ class ACRequiredEMTest : AIClassificationEMTestBase() {`
`31`	`31`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`32`	`32`	`}`
`33`	`33`
`34`		`- @Disabled`
	`34`	`+`
`35`	`35`	`@Test`
`36`	`36`	`fun testRunGaussian(){`
`37`	`37`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ class ACZeroOrOneEMTest : AIClassificationEMTestBase() {`
`35`	`35`	`testRunEM(AIResponseClassifierModel.DETERMINISTIC)`
`36`	`36`	`}`
`37`	`37`
`38`		`- @Disabled`
	`38`	`+`
`39`	`39`	`@Test`
`40`	`40`	`fun testRunGaussian(){`
`41`	`41`	`testRunEM(AIResponseClassifierModel.GAUSSIAN)`