Merge pull request #1375 from WebFuzzing/core-driver-it

arcuri82 · web-flow · commit f10a54afc8b4 · 2025-11-04T09:51:18.000+01:00
fixed and re-enabled core-driver-it tests
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -40,10 +40,9 @@ env:
   # https://github.com/WebFuzzing/EvoMaster/issues/447
   release-jdk: 21
   build-jdk: 17
-#  latest-jdk: 21
   retention-days: 5
   debug: false  # put to true if need to debug a specific test
-  debugTestName: "org.evomaster.e2etests.spring.graphql.db.tree.DbTreeEMTest" # replace with test to debug
+  debugTestName: "com.foo.base.BaseIT" # replace with test to debug
 
 # This build is quite expensive (some hours), so we run it whole only on some JVM versions and OSs.
 # For the moment, we need to support JVM 8 and all following LTS versions (e.g, 11 and  17).
@@ -84,7 +83,7 @@ jobs:
       - name: Debug Test Case
         env:
           CI_env: GithubAction
-        run: mvn test  -Dtest=${{env.debugTestName}} -Dsurefire.failIfNoSpecifiedTests=false  -DredirectTestOutputToFile=false
+        run: mvn -B install  -Dtest=${{env.debugTestName}} -Dsurefire.failIfNoSpecifiedTests=false  -DredirectTestOutputToFile=false
 
   full-build-base:
     runs-on: ubuntu-latest
@@ -122,7 +121,7 @@ jobs:
         env:
           NODE_OPTIONS: "--max_old_space_size=8000"
         with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
+          report_paths: '**/target/*-reports/TEST-*.xml'
       # Upload coverage results
       # As of 2024, looks like a rate limiter is breaking the build.
       # As such, now we only upload from master branch, hoping it is enough.
diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/ExternalSutController.java
@@ -93,7 +93,12 @@ public final void setupForGeneratedTest(){
         //In the past, we configured P6Spy here
     }
 
-    public void setNeedsJdk17Options(boolean needsJdk17Options) {
+    /**
+     * Unfortunately, the use of --add-opens in JDK 17+ has broken many things... :(
+     * If you are using JDK 17 or above, you must set this to true
+     * @param needsJdk17Options
+     */
+    public final void setNeedsJdk17Options(boolean needsJdk17Options) {
         this.needsJdk17Options = needsJdk17Options;
     }
 
diff --git a/core-tests/integration-tests/core-driver-it/pom.xml b/core-tests/integration-tests/core-driver-it/pom.xml
@@ -31,10 +31,10 @@
             <groupId>org.springdoc</groupId>
             <artifactId>springdoc-openapi-ui</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.springdoc</groupId>
-            <artifactId>springdoc-openapi-security</artifactId>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>org.springdoc</groupId>-->
+<!--            <artifactId>springdoc-openapi-security</artifactId>-->
+<!--        </dependency>-->
         <dependency>
             <groupId>org.springdoc</groupId>
             <artifactId>springdoc-openapi-kotlin</artifactId>
@@ -46,6 +46,12 @@
             <artifactId>evomaster-client-java-controller</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.evomaster</groupId>
+            <artifactId>evomaster-client-java-instrumentation</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>org.evomaster</groupId>
@@ -97,6 +103,14 @@
                     <finalName>base</finalName>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <!--  Fucking surefire is running the IT tests for some fucking reasons... -->
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
diff --git a/core-tests/integration-tests/core-driver-it/src/test/kotlin/com/foo/base/BaseIT.kt b/core-tests/integration-tests/core-driver-it/src/test/kotlin/com/foo/base/BaseIT.kt
@@ -3,14 +3,12 @@ package com.foo.base
 import org.evomaster.client.java.controller.InstrumentedSutStarter
 import org.evomaster.client.java.controller.api.dto.ActionDto
 import org.evomaster.core.remote.service.RemoteController
-import org.evomaster.ci.utils.CIUtils
 import org.evomaster.core.remote.service.RemoteControllerImplementation
 import org.junit.jupiter.api.*
 import org.junit.jupiter.api.Assertions.*
 import java.io.File
 
 
-@Disabled("No CI (Travis, CircleCI and GitHub) likes this test... :( ")
 class BaseIT {
 
 
@@ -19,26 +17,35 @@ class BaseIT {
         private val starter = InstrumentedSutStarter(driver)
         private lateinit var remote: RemoteController
 
+
         private fun setupJarAgent(){
 
-            val path = File("../client-java/instrumentation/target").walk()
+            val folder = File("../../../client-java/instrumentation/target").absoluteFile
+            if(!folder.exists()){
+                throw IllegalStateException("Target folder does not exist: ${folder.absolutePath}")
+            }
+
+            val files = folder.listFiles()
+
+            val path = files
                     .filter { it.name.endsWith(".jar") }
                     .find {
                         it.name.matches(Regex("evomaster-client-java-instrumentation-\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?\\.jar"))
-                    }!!
-                    .absolutePath
+                    }?.absolutePath
+            if(path == null) {
+                val names = files.map { it.name }
+                throw IllegalStateException("evomaster-client-java-instrumentation jar file not found in target folder: ${folder.absolutePath}." +
+                            " | Content: ${names.joinToString(", ")}")
+            }
 
             System.setProperty("evomaster.instrumentation.jar.path", path)
         }
 
         @JvmStatic
         @BeforeAll
         fun beforeAll() {
-            //Travis and CircleCI do not like this test...
-            CIUtils.skipIfOnTravis()
-            CIUtils.skipIfOnCircleCI()
-
             setupJarAgent()
+            driver.setNeedsJdk17Options(true)
             driver.controllerPort = 0
             starter.start()
             remote = RemoteControllerImplementation("localhost", driver.controllerServerPort, false, false)
@@ -58,45 +65,66 @@ class BaseIT {
 
     @Test
     fun testSearchCommands(){
-        remote.startANewSearch()
-        remote.startSUT()
+        val startedNewSearch = remote.startANewSearch()
+        assertTrue(startedNewSearch, "Failed to start new search")
+
+        val startedSUT = remote.startSUT()
+        assertTrue(startedSUT, "Failed to start SUT")
 
-        remote.registerNewAction(ActionDto().apply { index = 0 })
+        val actionRegistered = remote.registerNewAction(ActionDto().apply { index = 0 })
+        assertTrue(actionRegistered, "Failed to register action")
 
         val results = remote.getTestResults()
-        assertNotNull(results)
+        assertNotNull(results, "Failed to get results")
         assertEquals(0, results!!.targets.size)
     }
 
     @Test
     fun testRestart(){
 
         //make sure it is started
-        assertTrue(remote.startSUT())
+        assertTrue(remote.startSUT(), "Failed to start SUT")
         var info = remote.getSutInfo()
-        assertNotNull(info)
+        assertNotNull(info, "Failed to get SUT info")
 
         //stop it
-        assertTrue(remote.stopSUT())
+        assertTrue(remote.stopSUT(), "Failed to stop SUT")
         info = remote.getSutInfo()
-        assertNull(info)
+        assertNull(info, "Failed to get SUT info after stop")
 
+        val n = 3
+        for(i in 0 until n){
+            driver.sutPort++
 
-        //start it again
-        driver.sutPort++ //let's try to avoid issue with TCP port taking too long to be released
-        assertTrue(remote.startSUT())
-        info = remote.getSutInfo()
-        assertNotNull(info)
+            val started = remote.startSUT()
+            val before = remote.getSutInfo()
+            val stopped = remote.stopSUT()
+            val after = remote.getSutInfo()
 
-        //stop it
-        assertTrue(remote.stopSUT())
-        info = remote.getSutInfo()
-        assertNull(info)
+            if(started && stopped && before != null && after == null) {
+                //all good
+                return
+            }
+        }
 
-        //start it again
-        driver.sutPort++
-        assertTrue(remote.startSUT())
-        info = remote.getSutInfo()
-        assertNotNull(info)
+        fail<Any>("Failed to start/stop SUT with $n attempts")
+
+        //REFACTORED due to possible issues with port collisions
+//        //start it again
+//        driver.sutPort++ //let's try to avoid issue with TCP port taking too long to be released
+//        assertTrue(remote.startSUT(), "Failed to re-start SUT on port ${driver.sutPort}")
+//        info = remote.getSutInfo()
+//        assertNotNull(info, "Failed to get SUT info after re-start")
+//
+//        //stop it
+//        assertTrue(remote.stopSUT(), "Failed to re-stop SUT")
+//        info = remote.getSutInfo()
+//        assertNull(info, "Failed to get SUT info after re-stop")
+//
+//        //start it again
+//        driver.sutPort++
+//        assertTrue(remote.startSUT(), "Failed to re-start SUT in 3rd time on port ${driver.sutPort}")
+//        info = remote.getSutInfo()
+//        assertNotNull(info)
     }
 }
diff --git a/core-tests/pom.xml b/core-tests/pom.xml
@@ -17,6 +17,10 @@
         <module>integration-tests</module>
     </modules>
 
+    <properties>
+        <springdoc.version>1.6.9</springdoc.version>
+    </properties>
+
 
 
 
diff --git a/pom.xml b/pom.xml
@@ -124,7 +124,6 @@
         <springboot.version>2.5.4</springboot.version>
         <springboot3.version>3.1.5</springboot3.version>
         <springfox.version>3.0.0</springfox.version>
-        <springdoc.version>1.6.9</springdoc.version>
         <jetty.version>9.4.29.v20200521</jetty.version>
         <slf4j.version>1.7.24</slf4j.version>
         <jersey.version>2.33</jersey.version>
@@ -929,24 +928,27 @@
                             https://bugs.openjdk.java.net/browse/JDK-8266851
                         -->
                         <argLine>@{argLine} -ea -Xms1024m -Xmx4096m -Xss4m -Dfile.encoding=UTF-8 -Djdk.attach.allowAttachSelf=true -Duser.language=en -Duser.country=GB ${addOpens}</argLine>
-                        <!-- Needed for some weird bug in JDK used in CircleCI-->
-                        <!--  Commented out, as no longer using CircleCI-->
-                        <!--  <useSystemClassLoader>false</useSystemClassLoader>-->
                         <runOrder>alphabetical</runOrder>
+                        <excludes>
+                            <!-- Do NOT run integration tests with surefire... use failsafe!!!-->
+                            <exclude>**/*IT.java</exclude>
+                            <exclude>**/*IT.kt</exclude>
+                        </excludes>
                     </configuration>
                 </plugin>
 
                 <!-- To run integration tests -->
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-failsafe-plugin</artifactId>
-                    <version>3.0.0-M4</version>
+                    <version>3.5.4</version>
                     <configuration>
                         <forkCount>1</forkCount>
-                        <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                        <redirectTestOutputToFile>${redirectTestOutputToFile}</redirectTestOutputToFile>
                         <rerunFailingTestsCount>2</rerunFailingTestsCount>
-                        <argLine>@{argLine} -ea -Xms1024m -Xmx4096m -Djdk.attach.allowAttachSelf=true ${addOpens}</argLine>
-                        <useSystemClassLoader>false</useSystemClassLoader>
+                        <trimStackTrace>false</trimStackTrace>
+                        <argLine>@{argLine} -ea -Xms1024m -Xmx4096m -Djdk.attach.allowAttachSelf=true -Duser.language=en -Duser.country=GB ${addOpens}</argLine>
+                        <runOrder>alphabetical</runOrder>
                     </configuration>
                     <executions>
                         <execution>

Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,12 @@ public final void setupForGeneratedTest(){`
`93`	`93`	`//In the past, we configured P6Spy here`
`94`	`94`	`}`
`95`	`95`
`96`		`- public void setNeedsJdk17Options(boolean needsJdk17Options) {`
	`96`	`+ /**`
	`97`	`+ * Unfortunately, the use of --add-opens in JDK 17+ has broken many things... :(`
	`98`	`+ * If you are using JDK 17 or above, you must set this to true`
	`99`	`+ * @param needsJdk17Options`
	`100`	`+ */`
	`101`	`+ public final void setNeedsJdk17Options(boolean needsJdk17Options) {`
`97`	`102`	`this.needsJdk17Options = needsJdk17Options;`
`98`	`103`	`}`
`99`	`104`