support handleFlaky with tests

Author: man-zhang

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -207,7 +207,7 @@ protected void runTestHandlingFlaky(
             Consumer<List<String>> lambda,
             int timeoutMinutes) throws Throwable{
 
-        runTestHandlingFlaky(outputFolderName, fullClassName, null, iterations, createTests, lambda, timeoutMinutes);
+        runTestHandlingFlaky(outputFolderName, fullClassName, null, iterations, createTests, false, lambda, timeoutMinutes);
     }
 
     protected void runTestHandlingFlaky(
@@ -216,6 +216,7 @@ protected void runTestHandlingFlaky(
             List<String> terminations,
             int iterations,
             boolean createTests,
+            boolean handleFlakyWithTests,
             Consumer<List<String>> lambda,
             int timeoutMinutes) throws Throwable{
         List<ClassName> classNames = new ArrayList<>();
@@ -246,6 +247,9 @@ protected void runTestHandlingFlaky(
                         List<String> args = getArgsWithCompilation(iterations, outputFolderName, className, createTests, finalSplitType, "FALSE");
                         defaultSeed++;
                         lambda.accept(new ArrayList<>(args));
+
+                        if (createTests && handleFlakyWithTests)
+                            runTests(outputFolderName, fullClassName, terminations);
                     }
             );
         });
@@ -257,7 +261,7 @@ protected void runTestHandlingFlakyAndCompilation(
             int timeoutMinutes,
             Consumer<List<String>> lambda) throws Throwable {
 
-        runTestHandlingFlakyAndCompilation(label, "org.bar."+label, null,  iterations, true, lambda, timeoutMinutes);
+        runTestHandlingFlakyAndCompilation(label, "org.bar."+label, null,  iterations, true, false, lambda, timeoutMinutes);
     }
 
     protected void runTestHandlingFlakyAndCompilation(
@@ -275,7 +279,17 @@ protected void runTestHandlingFlakyAndCompilation(
             int iterations,
             Consumer<List<String>> lambda) throws Throwable {
 
-        runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, null, iterations, true, lambda, 3);
+        runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, null, iterations, true, false, lambda, 3);
+    }
+
+    protected void runTestHandlingFlakyAndCompilation(
+            String outputFolderName,
+            String fullClassName,
+            int iterations,
+            boolean handleFlakyWithTests,
+            Consumer<List<String>> lambda) throws Throwable {
+
+        runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, null, iterations, true, handleFlakyWithTests, lambda, 3);
     }
 
     protected void runTestHandlingFlakyAndCompilation(
@@ -285,7 +299,7 @@ protected void runTestHandlingFlakyAndCompilation(
             int iterations,
             Consumer<List<String>> lambda) throws Throwable {
 
-        runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, terminations, iterations, true, lambda, 3);
+        runTestHandlingFlakyAndCompilation(outputFolderName, fullClassName, terminations, iterations, true, false, lambda, 3);
     }
 
     protected void runTestHandlingFlakyAndCompilation(
@@ -294,24 +308,31 @@ protected void runTestHandlingFlakyAndCompilation(
             List<String> terminations,
             int iterations,
             boolean createTests,
+            boolean handleFlakyWithTests,
             Consumer<List<String>> lambda,
             int timeoutMinutes) throws Throwable {
 
-        runTestHandlingFlaky(outputFolderName, fullClassName, terminations, iterations, createTests,lambda, timeoutMinutes);
+        runTestHandlingFlaky(outputFolderName, fullClassName, terminations, iterations, createTests, false, lambda, timeoutMinutes);
 
         if (terminations == null) terminations = Arrays.asList("");
         //BMR: this is where I should handle multiples???
-        if (createTests){
-            MethodReplacementPreserveSemantics.shouldPreserveSemantics = true;
+        if (createTests && !handleFlakyWithTests){
+            runTests(outputFolderName, fullClassName, terminations);
+        }
+    }
 
-            for (String termination : terminations) {
-                assertTimeoutPreemptively(Duration.ofMinutes(2), () -> {
-                    ClassName className = new ClassName(fullClassName + termination);
-                    clearCompiledFiles(className);
-                    //the first one goes through, but for the second generated files appear to not be clean.
-                    compileRunAndVerifyTests(outputFolderName, className);
-                });
-            }
+    protected void runTests(String outputFolderName,
+                            String fullClassName,
+                            List<String> terminations){
+        MethodReplacementPreserveSemantics.shouldPreserveSemantics = true;
+
+        for (String termination : terminations) {
+            assertTimeoutPreemptively(Duration.ofMinutes(2), () -> {
+                ClassName className = new ClassName(fullClassName + termination);
+                clearCompiledFiles(className);
+                //the first one goes through, but for the second generated files appear to not be clean.
+                compileRunAndVerifyTests(outputFolderName, className);
+            });
         }
     }
 

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt

@@ -29,7 +29,8 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() {
         runTestHandlingFlakyAndCompilation(
             outputFolder,
             outputClass,
-            2000
+            2000,
+            true,
         ) { args: MutableList<String> ->