Merge branch 'master' into feature/os-distance-operations

Author: arcuri82

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ env:
   release-jdk: 21
   build-jdk: 17
   retention-days: 5
-  debug: true  # put to true if need to debug a specific test
+  debug: false  # put to true if need to debug a specific test
   debugTestName: "GeneRandomizedTest" # replace with test to debug
 
 # This build is quite expensive (some hours), so we run it whole only on some JVM versions and OSs.
@@ -62,6 +62,11 @@ jobs:
      - name: Set skip-jobs variable
        id: set-output
        run: echo "debug=${{ env.debug }}" >> $GITHUB_OUTPUT
+     - name: Verify Docker is working fine
+       run: | 
+        timeout 30s bash -c 'until docker info; do sleep 1; done' 
+        docker version
+        docker info 
 
 
   debug:

client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java

@@ -1953,6 +1953,7 @@ public final String readFileAsStringFromTestResource(String fileName){
                 .lines().collect(Collectors.joining(System.lineSeparator()));
     }
 
+
     @Override
     public Map<Class, Integer> getExceptionImportanceLevels() {
         return null;

client-java/test-utils-java/src/main/java/org/evomaster/test/utils/EMTestUtils.java

@@ -13,6 +13,8 @@ is used in the EvoMaster Core (eg, when making HTTP calls) and
  */
 
 import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * Class containing utility functions that can be used in the
@@ -137,4 +139,67 @@ public static boolean isValidURIorEmpty(String uri){
             return false;
         }
     }
+
+    /**
+     * Resolves the absolute path to the Java executable using the given
+     * JDK environment variable name.
+     *
+     * <p>This method expects the environment variable (e.g. {@code JAVA_HOME})
+     * to point to a JDK installation directory. It appends {@code "bin"} and
+     * {@code "java"} to construct the full path to the Java executable.</p>
+     *
+     *
+     * @param jdkEnvVarName the name of the JDK environment variable
+     *                      (e.g. {@code "JAVA_HOME"})
+     * @return the absolute path to the Java executable as a String
+     * @throws RuntimeException if the environment variable is not defined or empty
+     */
+    public static String extractJDKPathWithEnvVarName(String jdkEnvVarName){
+        return extractPathWithEnvVar(jdkEnvVarName, "bin", "java").toString();
+    }
+
+    /**
+     * Resolves the absolute path to a System-Under-Test (SUT) JAR file
+     * using environment variables.
+     *
+     *
+     * @param sutDistEnvVarName the environment variable that contains the base
+     *                          directory of the SUT distribution
+     * @param sutJarEnvVarName the name of the JAR file (or relative path inside the distribution)
+     * @return the absolute path to the SUT JAR file as a String
+     * @throws RuntimeException if the distribution environment variable is not defined or empty
+     */
+    public static String extractSutJarNameWithEnvVarName(String sutDistEnvVarName, String sutJarEnvVarName){
+        return extractPathWithEnvVar(sutDistEnvVarName, sutJarEnvVarName).toString();
+    }
+
+    /**
+     * Resolves an absolute {@link Path} using the value of a given environment variable
+     * as the base directory and appending additional path segments.
+     *
+     * <p>For example, if {@code envVarName} is {@code "JAVA_HOME"} and
+     * {@code others} contains {@code "bin", "java"}, this method will return:</p>
+     *
+     * <pre>
+     * $JAVA_HOME/bin/java   (Linux/macOS)
+     * %JAVA_HOME%\bin\java  (Windows)
+     * </pre>
+     *
+     * <p>The resulting path is converted to an absolute path.</p>
+     *
+     * @param envVarName the name of the environment variable (e.g. {@code "JAVA_HOME"})
+     * @param others additional path segments to append to the environment variable path
+     * @return the resolved absolute {@link Path}
+     * @throws RuntimeException if the environment variable is not defined or empty
+     */
+    private static Path extractPathWithEnvVar(String envVarName, String... others){
+        String javaHome = System.getenv(envVarName);
+
+        if (javaHome == null || javaHome.isEmpty()) {
+            throw new IllegalArgumentException("Environment variable does not seem to be defined: " + envVarName);
+        }
+
+        Path javaExecutable = Paths.get(javaHome, others);
+        return javaExecutable.toAbsolutePath();
+    }
 }

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -807,6 +807,18 @@ class EMConfig {
         if(dockerLocalhost && !runningInDocker){
             throw ConfigProblemException("Specifying 'dockerLocalhost' only makes sense when running EvoMaster inside Docker.")
         }
+
+        if(useEnvVarsForPathInTests){
+            if(problemType != ProblemType.DEFAULT && problemType != ProblemType.REST)
+                throw ConfigProblemException("'useEnvVarsForPathInTests' can be applied only for REST problem.")
+
+            if (jdkEnvVarName.isEmpty())
+                throw ConfigProblemException("'jdkEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.")
+            if (sutDistEnvVarName.isEmpty())
+                throw ConfigProblemException("'sutDistEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.")
+            if (sutJarEnvVarName.isEmpty())
+                throw ConfigProblemException("'sutJarEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.")
+        }
     }
 
     private fun checkPropertyConstraints(m: KMutableProperty<*>) {
@@ -2636,6 +2648,26 @@ class EMConfig {
             "Note that flakiness is now supported only for fuzzing REST APIs")
     var handleFlakiness = false
 
+    @Experimental
+    @Cfg("Use environment variables to define the paths required by External Drivers. " +
+            "This is necessary when the generated tests are executed on the different machine. " +
+            "Note that this setting only affects the generated test cases.")
+    var useEnvVarsForPathInTests = false
+
+    @Experimental
+    @Cfg("Specify name of the environment variable that provides the JDK installation path. " +
+            "Note that the executable path will be resolved by appending 'bin/java'.")
+    var jdkEnvVarName = ""
+
+    @Experimental
+    @Cfg("Specify name of the environment variable that provides the the base distribution directory of the " +
+            "SUT, e.g., 'dist' directory of WFD.")
+    var sutDistEnvVarName = ""
+
+    @Experimental
+    @Cfg("Specifies the name of the SUT JAR file that will be used together with `sutDistEnvVarName` to resolve the full SUT JAR path.")
+    var sutJarEnvVarName = ""
+
     @Experimental
     @Cfg("Specify a method to select the first external service spoof IP address.")
     var externalServiceIPSelectionStrategy = ExternalServiceIPSelectionStrategy.NONE

core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt

@@ -63,6 +63,10 @@ class TestSuiteWriter {
         private const val fixture = "_fixture"
         private const val browser = "browser"
         private var containsDtos = false
+
+        private const val EXTRACT_JDK_PATH_ENV_VAR_METHOD = "extractJDKPathWithEnvVarName"
+        private const val EXTRACT_SUT_PATH_ENV_VAR_METHOD = "extractSutJarNameWithEnvVarName"
+
     }
 
     @Inject
@@ -609,6 +613,10 @@ class TestSuiteWriter {
     }
 
     private fun getJavaCommand(): String {
+        if (config.useEnvVarsForPathInTests){
+            return ".setJavaCommand($EXTRACT_JDK_PATH_ENV_VAR_METHOD(\"${config.jdkEnvVarName}\"))"
+        }
+
         if (config.javaCommand != "java") {
             val java = config.javaCommand.replace("\\", "\\\\")
             return ".setJavaCommand(\"$java\")"
@@ -625,8 +633,12 @@ class TestSuiteWriter {
 
         val wireMockServers = getActiveWireMockServers()
 
-        val executable = if (controllerInput.isNullOrBlank()) ""
-        else "\"$controllerInput\"".replace("\\", "\\\\")
+        val executable = if (config.useEnvVarsForPathInTests){
+            "$EXTRACT_SUT_PATH_ENV_VAR_METHOD(\"${config.sutDistEnvVarName}\", \"${config.sutJarEnvVarName}\")"
+        }else{
+            if (controllerInput.isNullOrBlank()) ""
+            else "\"$controllerInput\"".replace("\\", "\\\\")
+        }
 
         if (config.outputFormat.isJava()) {
             if (!config.blackBox || config.bbExperiments) {

core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt

@@ -719,4 +719,25 @@ internal class EMConfigTest{
         assertEquals(config.maxTestCaseNameLength, different)
     }
 
+
+    @Test
+    fun testUseEnvVarsForPathInTests(){
+
+        val parser = EMConfig.getOptionParser()
+        parser.recognizedOptions()["useEnvVarsForPathInTests"] ?: throw Exception("Cannot find option")
+
+        val config = EMConfig()
+        val optionAllEmpty = parser.parse("--useEnvVarsForPathInTests", "true")
+        assertThrows(Exception::class.java, {config.updateProperties(optionAllEmpty)})
+
+        val optionOneMissing = parser.parse("--useEnvVarsForPathInTests", "true", "--jdkEnvVarName", "JDK_HOME", "--sutDistEnvVarName", "WFC_HOME")
+        assertThrows(Exception::class.java, {config.updateProperties(optionOneMissing)})
+
+        val optionAllSpecified = parser.parse("--useEnvVarsForPathInTests", "true", "--jdkEnvVarName", "JDK_HOME", "--sutDistEnvVarName", "WFC_HOME", "--sutJarEnvVarName", "sut-jar.jar")
+        assertDoesNotThrow ({config.updateProperties(optionAllSpecified)})
+
+        val optionAllSpecifiedNotRest = parser.parse("--useEnvVarsForPathInTests", "true", "--jdkEnvVarName", "JDK_HOME", "--sutDistEnvVarName", "WFC_HOME", "--sutJarEnvVarName", "sut-jar.jar", "--problemType", "RPC")
+        assertThrows (Exception::class.java,{config.updateProperties(optionAllSpecifiedNotRest)})
+
+    }
 }

core/src/test/kotlin/org/evomaster/core/output/service/TestSuiteWriterTest.kt

@@ -149,6 +149,59 @@ class TestSuiteWriterTest{
         assertTrue(generatedUtils == PyLoader::class.java.getResource("/${TestSuiteWriter.pythonUtilsFilename}").readText())
     }
 
+    @Test
+    fun testEmptySuiteWithEnabledEnvVar(){
+
+        val injector = getInjector()
+
+        val config = injector.getInstance(EMConfig::class.java)
+        config.createTests = true
+        config.outputFormat = OutputFormat.KOTLIN_JUNIT_5
+        config.outputFolder = "$baseTargetFolder/empty_suite_use_env_var"
+        config.outputFilePrefix = "Foo_testEmptySuiteUseEnvVar"
+        config.outputFileSuffix = ""
+        config.useEnvVarsForPathInTests = true
+        config.jdkEnvVarName = "JAVA_HOME_FAKE"
+        config.sutDistEnvVarName = "WFC_HOME_FAKE"
+        config.sutJarEnvVarName = "fake-sut.jar"
+
+        val solution = getEmptySolution(config)
+
+
+        //make sure we delete any existing folder from previous test runs
+        val srcFolder = File(config.outputFolder)
+        srcFolder.deleteRecursively()
+
+        //this is what used by Maven and IntelliJ
+        val testClassFolder = File("target/test-classes")
+        val expectedCompiledFile = testClassFolder.toPath()
+            .resolve("${config.outputFilePrefix}.class")
+            .toFile()
+        expectedCompiledFile.delete()
+        assertFalse(expectedCompiledFile.exists())
+
+
+        val writer = injector.getInstance(TestSuiteWriter::class.java)
+
+
+        //write the test suite
+        writer.writeTests(solution, FakeController::class.qualifiedName!!, null)
+
+        val generatedTest = Paths.get("${config.outputFolder}/${config.outputFilePrefix}.kt")
+        assertTrue(Files.exists(generatedTest))
+
+        /*
+            here, we only check the generated in text
+            as it requires to use method in SutHandler
+         */
+        val testContent = String(Files.readAllBytes(generatedTest))
+        val expectedJdkPath = "org.evomaster.core.output.service.FakeController(extractSutJarNameWithEnvVarName(\"${config.sutDistEnvVarName}\", \"${config.sutJarEnvVarName}\"))"
+        val expectedSutPath = ".setJavaCommand(extractJDKPathWithEnvVarName(\"${config.jdkEnvVarName}\"))"
+
+        assertTrue(testContent.contains(expectedJdkPath))
+        assertTrue(testContent.contains(expectedSutPath))
+    }
+
     private fun getEmptySolution(config: EMConfig): Solution<RestIndividual> {
         return Solution<RestIndividual>(
             mutableListOf(),

docs/options.md

@@ -286,6 +286,7 @@ There are 3 types of options:
 |`instrumentMR_NET`| __Boolean__. Execute instrumentation for method replace with category NET. Note: this applies only for languages in which instrumentation is applied at runtime, like Java/Kotlin on the JVM. *Default value*: `false`.|
 |`instrumentMR_OPENSEARCH`| __Boolean__. Execute instrumentation for method replace with category OPENSEARCH. Note: this applies only for languages in which instrumentation is applied at runtime, like Java/Kotlin on the JVM. *Default value*: `false`.|
 |`instrumentMR_REDIS`| __Boolean__. Execute instrumentation for method replace with category REDIS. Note: this applies only for languages in which instrumentation is applied at runtime, like Java/Kotlin on the JVM. *Default value*: `false`.|
+|`jdkEnvVarName`| __String__. Specify name of the environment variable that provides the JDK installation path. Note that the executable path will be resolved by appending 'bin/java'. *Default value*: `""`.|
 |`languageModelConnector`| __Boolean__. Enable language model connector. *Default value*: `false`.|
 |`languageModelConnectorNumberOfThreads`| __Int__. Number of threads for language model connector. No more threads than numbers of processors will be used. *Constraints*: `min=1.0`. *Default value*: `2`.|
 |`languageModelName`| __String__. Large-language model name as listed in Ollama. *Default value*: `llama3.2:latest`.|
@@ -321,10 +322,13 @@ There are 3 types of options:
 |`ssrf`| __Boolean__. To apply SSRF detection as part of security testing. *Depends on*: `security=true`. *Default value*: `false`.|
 |`structureMutationProFS`| __Double__. Specify a probability of applying structure mutator during the focused search. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`structureMutationProbStrategy`| __Enum__. Specify a strategy to handle a probability of applying structure mutator during the focused search. *Valid values*: `SPECIFIED, SPECIFIED_FS, DPC_TO_SPECIFIED_BEFORE_FS, DPC_TO_SPECIFIED_AFTER_FS, ADAPTIVE_WITH_IMPACT`. *Default value*: `SPECIFIED`.|
+|`sutDistEnvVarName`| __String__. Specify name of the environment variable that provides the the base distribution directory of the SUT, e.g., 'dist' directory of WFD. *Default value*: `""`.|
+|`sutJarEnvVarName`| __String__. Specifies the name of the SUT JAR file that will be used together with `sutDistEnvVarName` to resolve the full SUT JAR path. *Default value*: `""`.|
 |`taintForceSelectionOfGenesWithSpecialization`| __Boolean__. During mutation, force the mutation of genes that have newly discovered specialization from previous fitness evaluations, based on taint analysis. *Default value*: `false`.|
 |`targetHeuristicsFile`| __String__. Where the target heuristic values file (if any) is going to be written (in CSV format). It is only used when processFormat is TARGET_HEURISTIC. *Default value*: `targets.csv`.|
 |`testResourcePathToSaveMockedResponse`| __String__. Specify test resource path where to save mocked responses as separated files. *Default value*: `""`.|
 |`thresholdDistanceForDataPool`| __Int__. Threshold of Levenshtein Distance for key-matching in Data Pool. *Constraints*: `min=0.0`. *Default value*: `2`.|
+|`useEnvVarsForPathInTests`| __Boolean__. Use environment variables to define the paths required by External Drivers. This is necessary when the generated tests are executed on the different machine. Note that this setting only affects the generated test cases. *Default value*: `false`.|
 |`useGlobalTaintInfoProbability`| __Double__. When sampling new individual, check whether to use already existing info on tainted values. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|
 |`useInsertionForSqlHeuristics`| __Boolean__. Specify whether insertions should be used to calculate SQL heuristics instead of retrieving data from real databases. *Default value*: `false`.|
 |`useTestMethodOrder`| __Boolean__. Adds TestMethodOrder annotation for JUnit 5 tests. *Default value*: `false`.|

pom.xml

@@ -122,7 +122,7 @@
         -->
         <jackson.version>2.14.3</jackson.version>
         <asm.version>9.9</asm.version>
-        <testcontainers.version>1.21.0</testcontainers.version>
+        <testcontainers.version>1.21.4</testcontainers.version>
         <testcontainers.mockerserver.version>1.17.2</testcontainers.mockerserver.version>
         <mockserver.client.version>5.13.2</mockserver.client.version>
 <!--        <selenium.version>4.31.0</selenium.version>-->