From 8d271a6fc778ac8490779b269d3c0d7f5bcb70b1 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 3 Dec 2024 15:42:01 +0100 Subject: [PATCH 01/50] starting preparing E2E tests --- .../multidb/separatedschemas/EntityX.kt | 15 ++++++++ .../multidb/separatedschemas/EntityY.kt | 15 ++++++++ .../multidb/separatedschemas/RepositoryX.kt | 5 +++ .../multidb/separatedschemas/RepositoryY.kt | 5 +++ .../SeparatedSchemasApplication.kt | 23 +++++++++++ .../separatedschemas/SeparatedSchemasRest.kt | 38 +++++++++++++++++++ .../SeparatedSchemasController.kt | 20 ++++++++++ .../SeparatedSchemasEMTest.kt | 36 ++++++++++++++++++ 8 files changed, 157 insertions(+) create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt create mode 100644 e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt new file mode 100644 index 0000000000..b0e84d19fb --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityX.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Table +import javax.validation.constraints.NotNull + +@Entity +@Table(name = "EntityX", schema = "foo") +open class EntityX { + + @get:Id + @get:NotNull + open var id: String? = null +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt new file mode 100644 index 0000000000..30ed285600 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/EntityY.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Table +import javax.validation.constraints.NotNull + +@Entity +@Table(name = "EntityY", schema = "bar") +open class EntityY { + + @get:Id + @get:NotNull + open var id: String? = null +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt new file mode 100644 index 0000000000..517e74d726 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryX.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.data.repository.CrudRepository + +interface RepositoryX : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt new file mode 100644 index 0000000000..7783bbdac7 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/RepositoryY.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.data.repository.CrudRepository + +interface RepositoryY : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt new file mode 100644 index 0000000000..77a2129678 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasApplication.kt @@ -0,0 +1,23 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class SeparatedSchemasApplication { + + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(SeparatedSchemasApplication::class.java, *args) + } + } + + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt new file mode 100644 index 0000000000..26ac3417c5 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt @@ -0,0 +1,38 @@ +package com.foo.rest.examples.multidb.separatedschemas + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping(path = ["/api/separatedschemas"]) +@RestController +open class SeparatedSchemasRest( + private val repositoryX: RepositoryX, + private val repositoryY: RepositoryY +) { + + @GetMapping(path = ["/x/{id}"]) + open fun getX(@PathVariable id: String) : ResponseEntity { + + val found = repositoryX.findById(id) + if(found.isPresent) { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.notFound().build() + } + + @GetMapping(path = ["/y/{id}"]) + open fun getY(@PathVariable id: String) : ResponseEntity { + + val found = repositoryY.findById(id) + if(found.isPresent) { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.notFound().build() + } + +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt new file mode 100644 index 0000000000..08790d2b20 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt @@ -0,0 +1,20 @@ +package org.evomaster.driver.multidb.separatedschemas + +import com.foo.rest.examples.multidb.separatedschemas.SeparatedSchemasApplication +import org.evomaster.client.java.sql.SqlScriptRunner +import org.evomaster.driver.multidb.SpringController + +class SeparatedSchemasController : SpringController(SeparatedSchemasApplication::class.java){ + + + override fun resetStateOfSUT() { + SqlScriptRunner.execCommand(connectionIfExist, """ + CREATE SCHEMA IF NOT EXISTS foo; + CREATE SCHEMA IF NOT EXISTS bar; + CREATE TABLE IF NOT EXISTS foo.EntityX; + CREATE TABLE IF NOT EXISTS bar.EntityY; + """.trimIndent()) + + super.resetStateOfSUT() + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt new file mode 100644 index 0000000000..baa1309fb5 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt @@ -0,0 +1,36 @@ +package org.evomaster.e2etests.spring.multidb.separatedschemas + +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.sql.SqlScriptRunner +import org.evomaster.core.problem.rest.HttpVerb +import org.evomaster.driver.multidb.SpringController +import org.evomaster.e2etests.spring.multidb.MultiDbParameterizedE2ETemplate +import org.junit.jupiter.api.Assertions.assertTrue +import org.evomaster.driver.multidb.separatedschemas.SeparatedSchemasController + +/** + * Created by arcuri82 on 03-Mar-20. + */ +class SeparatedSchemasEMTest : MultiDbParameterizedE2ETemplate() { + + override fun instantiateNewController(): SpringController { + return SeparatedSchemasController() + } + + override fun runEM(databaseType: DatabaseType) { + + runTestHandlingFlakyAndCompilation( + "SeparatedSchemasEM_$databaseType", + 100 + ) { args: MutableList -> + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 404, "/api/separatedschemas/x/{id}", null) + assertHasAtLeastOne(solution, HttpVerb.GET, 404, "/api/separatedschemas/y/{id}", null) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/separatedschemas/x/{id}", "OK") + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/separatedschemas/y/{id}", "OK") + } + } +} \ No newline at end of file From fe202e69ca066605841e284f09ae92e928140103 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 4 Dec 2024 15:29:44 +0100 Subject: [PATCH 02/50] trying to get rid off JPA in new E2E --- .../rest/examples/multidb/base/BaseEntity.kt | 2 +- .../examples/multidb/base/BaseRepository.kt | 2 +- .../rest/examples/multidb/base/BaseRest.kt | 13 +++++++++--- .../separatedschemas/SeparatedSchemasRest.kt | 21 +++++++++++++------ .../driver/multidb/SpringController.kt | 6 ++++-- .../driver/multidb/base/BaseController.kt | 9 +++++++- .../SeparatedSchemasController.kt | 15 +++++-------- .../src/main/resources/application.yml | 10 +++++++++ .../src/main/resources/sql/base.sql | 5 +++++ .../main/resources/sql/separatedschemas.sql | 12 +++++++++++ 10 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/application.yml create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql create mode 100644 e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt index c0c8089187..23ca487e6f 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt @@ -4,7 +4,7 @@ import javax.persistence.Entity import javax.persistence.Id import javax.validation.constraints.NotNull -@Entity +//@Entity open class BaseEntity( @get:Id @get:NotNull diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt index 4e36e3a674..733bb2514b 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt @@ -2,4 +2,4 @@ package com.foo.rest.examples.multidb.base import org.springframework.data.repository.CrudRepository -interface BaseRepository : CrudRepository \ No newline at end of file +//interface BaseRepository : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index 3f9a38e097..c474984822 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,8 @@ package com.foo.rest.examples.multidb.base +import com.foo.rest.examples.multidb.separatedschemas.EntityX import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @@ -9,14 +11,19 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/base"]) @RestController open class BaseRest( - private val repository: BaseRepository + //private val repository: BaseRepository + private val jdbc: JdbcTemplate ) { @GetMapping(path = ["/{id}"]) open fun get(@PathVariable id: String) : ResponseEntity { - val found = repository.findById(id) - if(found.isPresent) { + + val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", BaseEntity::class.java) + +// val found = repository.findById(id) +// if(found.isPresent) { + if(x != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt index 26ac3417c5..f615488540 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/separatedschemas/SeparatedSchemasRest.kt @@ -1,6 +1,8 @@ package com.foo.rest.examples.multidb.separatedschemas import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.jdbc.core.queryForObject import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @@ -9,15 +11,19 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/separatedschemas"]) @RestController open class SeparatedSchemasRest( - private val repositoryX: RepositoryX, - private val repositoryY: RepositoryY + //private val repositoryX: RepositoryX, + //private val repositoryY: RepositoryY + private val jdbc: JdbcTemplate ) { @GetMapping(path = ["/x/{id}"]) open fun getX(@PathVariable id: String) : ResponseEntity { - val found = repositoryX.findById(id) - if(found.isPresent) { + val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", EntityX::class.java) + + //val found = repositoryX.findById(id) +// if(found.isPresent) { + if(x!=null) { return ResponseEntity.ok("OK") } @@ -27,8 +33,11 @@ open class SeparatedSchemasRest( @GetMapping(path = ["/y/{id}"]) open fun getY(@PathVariable id: String) : ResponseEntity { - val found = repositoryY.findById(id) - if(found.isPresent) { +// val found = repositoryY.findById(id) +// if(found.isPresent) { + val y = jdbc.queryForObject("SELECT * FROM bar.EntityY WHERE id = ?", EntityY::class.java, id) + + if(y != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt index 7e2dc51fd6..dc61b5b818 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/SpringController.kt @@ -50,6 +50,8 @@ abstract class SpringController(protected val applicationClass: Class<*>) : Embe //startSut() } + abstract fun extraSpringStartOptions() : List + override fun startSut(): String { //lot of problem if using same H2 instance. see: @@ -65,9 +67,9 @@ abstract class SpringController(protected val applicationClass: Class<*>) : Embe val commonSettings = arrayOf( "--server.port=0", "--spring.jpa.properties.hibernate.show_sql=true", - "--spring.jpa.hibernate.ddl-auto=create-drop", + "--spring.jpa.hibernate.ddl-auto=validate", "--spring.jmx.enabled=false" - ) + ).plus(extraSpringStartOptions()) ctx = when(databaseType) { DatabaseType.H2 -> { diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt index b5035f5851..bd6d808b7e 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/base/BaseController.kt @@ -3,4 +3,11 @@ package org.evomaster.driver.multidb.base import com.foo.rest.examples.multidb.base.BaseApplication import org.evomaster.driver.multidb.SpringController -class BaseController : SpringController(BaseApplication::class.java) \ No newline at end of file +class BaseController : SpringController(BaseApplication::class.java){ + + override fun extraSpringStartOptions(): List { + return listOf( + "--spring.datasource.schema=classpath:/sql/base.sql" + ) + } +} \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt index 08790d2b20..b72314750c 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/org/evomaster/driver/multidb/separatedschemas/SeparatedSchemasController.kt @@ -6,15 +6,10 @@ import org.evomaster.driver.multidb.SpringController class SeparatedSchemasController : SpringController(SeparatedSchemasApplication::class.java){ - - override fun resetStateOfSUT() { - SqlScriptRunner.execCommand(connectionIfExist, """ - CREATE SCHEMA IF NOT EXISTS foo; - CREATE SCHEMA IF NOT EXISTS bar; - CREATE TABLE IF NOT EXISTS foo.EntityX; - CREATE TABLE IF NOT EXISTS bar.EntityY; - """.trimIndent()) - - super.resetStateOfSUT() + override fun extraSpringStartOptions(): List { + return listOf( + "--spring.datasource.schema=classpath:/sql/separatedschemas.sql" + ) } + } \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/application.yml b/e2e-tests/spring-rest-multidb/src/main/resources/application.yml new file mode 100644 index 0000000000..2f3f7d6c13 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/application.yml @@ -0,0 +1,10 @@ +spring: + jmx: + enabled: false + jpa: + show-sql: true +server: + error: + include-stacktrace: always + include-exception: true + diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql new file mode 100644 index 0000000000..2146213bea --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS BaseEntity( + id VARCHAR(50) NOT NULL, + name VARCHAR(50) NOT NULL, + constraint pk_x primary key (id) +); \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql new file mode 100644 index 0000000000..735f711f33 --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/separatedschemas.sql @@ -0,0 +1,12 @@ +CREATE SCHEMA IF NOT EXISTS foo; +CREATE SCHEMA IF NOT EXISTS bar; + +CREATE TABLE IF NOT EXISTS foo.EntityX( + id VARCHAR(50) NOT NULL, + constraint pk_x primary key (id) +); + +CREATE TABLE IF NOT EXISTS bar.EntityY( + id VARCHAR(50) NOT NULL, + constraint pk_y primary key (id) +); \ No newline at end of file From 82e2f8753b2cf8f17245442b582d2c2d7b27090b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 11:10:43 +0100 Subject: [PATCH 03/50] refactoring and cleaning --- .../controller/internal/SutController.java | 25 ++++++++++++------- .../client/java/sql/DbInfoExtractor.java | 13 +++------- ...DatabaseDtoUtils.java => SqlDtoUtils.java} | 11 ++++++-- .../sql/internal/HeuristicsCalculator.java | 2 +- .../sql/internal/QueryResultTransformer.java | 7 +++--- .../client/java/sql/internal/SqlHandler.java | 2 +- .../java/sql/internal/SqlNameContext.java | 10 ++++---- .../org/evomaster/core/sql/SqlAction.kt | 7 ++++++ .../core/sql/SqlActionTransformer.kt | 2 +- .../rest/examples/multidb/base/BaseEntity.kt | 15 ----------- .../examples/multidb/base/BaseRepository.kt | 5 ---- .../rest/examples/multidb/base/BaseRest.kt | 6 +---- .../rest/examples/multidb/base/BaseTable.kt | 9 +++++++ .../src/main/resources/sql/base.sql | 4 ++- .../MultiDbParameterizedE2ETemplate.kt | 2 +- 15 files changed, 62 insertions(+), 58 deletions(-) rename client-java/sql/src/main/java/org/evomaster/client/java/sql/{internal/SqlDatabaseDtoUtils.java => SqlDtoUtils.java} (87%) delete mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt delete mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt create mode 100644 e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index adf5b193b3..3df05b9e11 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -374,9 +374,11 @@ private void computeSQLHeuristics(ExtraHeuristicsDto dto, List a if (!additionalInfoList.isEmpty()) { AdditionalInfo last = additionalInfoList.get(additionalInfoList.size() - 1); last.getSqlInfoData().stream().forEach(it -> { -// String sql = it.getCommand(); try { - final SqlExecutionLogDto sqlExecutionLogDto = new SqlExecutionLogDto(it.getSqlCommand(), it.hasThrownSqlException(), it.getExecutionTime()); + SqlExecutionLogDto sqlExecutionLogDto = new SqlExecutionLogDto( + it.getSqlCommand(), + it.hasThrownSqlException(), + it.getExecutionTime()); sqlHandler.handle(sqlExecutionLogDto); } catch (Exception e) { SimpleLogger.error("FAILED TO HANDLE SQL COMMAND: " + it.getSqlCommand()); @@ -406,7 +408,6 @@ private void computeSQLHeuristics(ExtraHeuristicsDto dto, List a accessedTables.addAll(sqlExecutionsDto.deletedData); accessedTables.addAll(sqlExecutionsDto.insertedData.keySet()); //accessedTables.addAll(executionDto.queriedData.keySet()); -// accessedTables.addAll(sqlExecutionsDto.insertedData.keySet()); accessedTables.addAll(sqlExecutionsDto.updatedData.keySet()); } } @@ -495,13 +496,13 @@ public final void cleanAccessedTables(){ // clean accessed tables Set tableDataToInit = null; if (!accessedTables.isEmpty()){ - List tablesToClean = new ArrayList<>(); - getTableToClean(accessedTables, tablesToClean); + List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); - }else - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + } else { + DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + } tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); } } @@ -582,7 +583,13 @@ public void addSuccessfulInitSqlInsertion(InsertionDto insertionDto){ successfulInitSqlInsertions.add(insertionDto); } - private void getTableToClean(List accessedTables, List tablesToClean){ + private List getTablesToClean(List accessedTables) { + List tablesToClean = new ArrayList<>(); + fillTablesToClean(accessedTables,tablesToClean); + return tablesToClean; + } + + private void fillTablesToClean(List accessedTables, List tablesToClean){ for (String t: accessedTables){ if (!findInCollectionIgnoreCase(t, tablesToClean).isPresent()){ if (findInMapIgnoreCase(t, fkMap).isPresent()){ @@ -591,7 +598,7 @@ private void getTableToClean(List accessedTables, List tablesToC findInCollectionIgnoreCase(t, e.getValue()).isPresent() && !findInCollectionIgnoreCase(e.getKey(), tablesToClean).isPresent()).map(Map.Entry::getKey).collect(Collectors.toList()); if (!fk.isEmpty()) - getTableToClean(fk, tablesToClean); + fillTablesToClean(fk, tablesToClean); }else { SimpleLogger.uniqueWarn("Cannot find the table "+t+" in ["+String.join(",", fkMap.keySet())+"]"); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 1a78c486c4..9186fa3888 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -544,12 +544,7 @@ private static void addConstraints(DbInfoDto schemaDto, List } } - private static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; - } - return dto.schema + "." + dto.name; - } + private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, DatabaseMetaData md, ResultSet tables, Set tableIds) throws SQLException { @@ -582,13 +577,13 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, tableDto.schema = tableSchema; tableDto.catalog = tableCatalog; - if (tableIds.contains(getId(tableDto))) { + if (tableIds.contains(SqlDtoUtils.getId(tableDto))) { /* * Perhaps we should throw a more specific exception than IllegalArgumentException */ - throw new IllegalArgumentException("Cannot handle repeated table " + getId(tableDto) + " in database"); + throw new IllegalArgumentException("Cannot handle repeated table " + SqlDtoUtils.getId(tableDto) + " in database"); } else { - tableIds.add(getId(tableDto)); + tableIds.add(SqlDtoUtils.getId(tableDto)); } Set pks = new HashSet<>(); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java similarity index 87% rename from client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java rename to client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java index 9f5417aed4..0f22c237dd 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java @@ -1,4 +1,4 @@ -package org.evomaster.client.java.sql.internal; +package org.evomaster.client.java.sql; import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; @@ -9,9 +9,16 @@ import java.util.Set; import java.util.stream.Collectors; -public class SqlDatabaseDtoUtils { +public class SqlDtoUtils { + public static String getId(TableDto dto){ + if(dto.schema == null){ + return dto.name; + } + return dto.schema + "." + dto.name; + } + /** * * @param filter specifies which column should be returned, null means all columns should be returned diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java index 8027fbd3bf..8237cfa184 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/HeuristicsCalculator.java @@ -432,7 +432,7 @@ private Object getValue(Expression exp, DataRow data) { Column column = (Column) exp; String name = column.getColumnName(); - String table = context.getTableName(column); + String table = context.getFullyQualifiedTableName(column); return data.getValueByName(name, table); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 44a7c20213..47f132a18b 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -6,6 +6,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; +import org.evomaster.client.java.sql.SqlDtoUtils; import org.evomaster.client.java.sql.VariableDescriptor; import java.time.Instant; @@ -129,7 +130,7 @@ public static List> cartesianProduct(List> values){ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto insertionDto, String tableName, Set relatedColumns, DbInfoDto dto, List existingQueryResults){ - List relatedColumnNames = SqlDatabaseDtoUtils.extractColumnNames(insertionDto, relatedColumns); + List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); if (!relatedColumnNames.isEmpty()){ QueryResult found = null; if (!existingQueryResults.isEmpty()) @@ -143,13 +144,13 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti if (foundTableSchema.isPresent()){ TableDto tableDto = foundTableSchema.get(); - List printableValue = SqlDatabaseDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); + List printableValue = SqlDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); assert printableValue.size() == relatedColumnNames.size(); List values = new ArrayList<>(); for (int i = 0; i < printableValue.size(); i++){ - ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); + ColumnDto columnDto = SqlDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); if (columnDto == null) throw new IllegalArgumentException("Cannot find column schema of "+ relatedColumnNames.get(i) + " in Table "+ tableName); values.add(getColumnValueBasedOnPrintableValue(printableValue.get(i), columnDto)); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java index 9f79c597b1..126bfc2b06 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java @@ -361,7 +361,7 @@ public Map> extractColumnsInvolvedInWhere(Statement statemen @Override public void visit(Column column) { - String tn = context.getTableName(column); + String tn = context.getFullyQualifiedTableName(column); if (tn.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)) { // TODO handle it properly when ll have support for sub-selects diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java index ec231749c6..5ee420f9ce 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java @@ -91,12 +91,12 @@ public boolean hasColumn(String tableName, String columnName){ * @param column a column object * @return the name of the table that this column belongs to */ - public String getTableName(Column column) { + public String getFullyQualifiedTableName(Column column) { Table table = column.getTable(); if (table != null) { - return tableAliases.getOrDefault(table.getName().toLowerCase(), table.getName().toLowerCase()); + return tableAliases.getOrDefault(table.getFullyQualifiedName().toLowerCase(), table.getFullyQualifiedName().toLowerCase()); } if(statement instanceof Select) { @@ -112,10 +112,10 @@ public String getTableName(Column column) { } } else if(statement instanceof Delete){ Delete delete = (Delete) statement; - return delete.getTable().getName().toLowerCase(); + return delete.getTable().getFullyQualifiedName().toLowerCase(); } else if(statement instanceof Update){ Update update = (Update) statement; - return update.getTable().getName().toLowerCase(); + return update.getTable().getFullyQualifiedName().toLowerCase(); }else { throw new IllegalArgumentException("Cannot handle table name for: " + statement); } @@ -131,7 +131,7 @@ private List getTableNamesInFrom() { FromItemVisitorAdapter visitor = new FromItemVisitorAdapter(){ @Override public void visit(Table table) { - names.add(table.getName().toLowerCase()); + names.add(table.getFullyQualifiedName().toLowerCase()); } }; diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt index edefe24ca3..0dffdc43dc 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt @@ -98,6 +98,13 @@ class SqlAction( } } + fun getFullQualifyingTableName() : String{ + if(openGroupName.isNullOrBlank()){ + return table.name + } + return "$openGroupName.${table.name}" + } + private val genes: List = (computedGenes ?: selectedColumns.map { SqlActionGeneBuilder().buildGene(id, table, it) } ).also { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt index f30eacfbc6..29277d4448 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt @@ -35,7 +35,7 @@ object SqlActionTransformer { } - val insertion = InsertionDto().apply { targetTable = action.table.name } + val insertion = InsertionDto().apply { targetTable = action.getFullQualifyingTableName() } for (g in action.seeTopGenes()) { if (g is SqlPrimaryKeyGene) { diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt deleted file mode 100644 index 23ca487e6f..0000000000 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseEntity.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.foo.rest.examples.multidb.base - -import javax.persistence.Entity -import javax.persistence.Id -import javax.validation.constraints.NotNull - -//@Entity -open class BaseEntity( - - @get:Id @get:NotNull - open var id: String? = null, - - @get:NotNull - open var name: String? = null, -) diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt deleted file mode 100644 index 733bb2514b..0000000000 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRepository.kt +++ /dev/null @@ -1,5 +0,0 @@ -package com.foo.rest.examples.multidb.base - -import org.springframework.data.repository.CrudRepository - -//interface BaseRepository : CrudRepository \ No newline at end of file diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index c474984822..6f35333041 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,5 @@ package com.foo.rest.examples.multidb.base -import com.foo.rest.examples.multidb.separatedschemas.EntityX import org.springframework.http.ResponseEntity import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping @@ -11,7 +10,6 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping(path = ["/api/base"]) @RestController open class BaseRest( - //private val repository: BaseRepository private val jdbc: JdbcTemplate ) { @@ -19,10 +17,8 @@ open class BaseRest( open fun get(@PathVariable id: String) : ResponseEntity { - val x = jdbc.queryForObject("SELECT * FROM foo.EntityX WHERE id = '$id'", BaseEntity::class.java) + val x = jdbc.queryForObject("SELECT * FROM foo.BaseTable WHERE id = '$id'", BaseTable::class.java) -// val found = repository.findById(id) -// if(found.isPresent) { if(x != null) { return ResponseEntity.ok("OK") } diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt new file mode 100644 index 0000000000..0a4129fead --- /dev/null +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseTable.kt @@ -0,0 +1,9 @@ +package com.foo.rest.examples.multidb.base + + +open class BaseTable( + + open var id: String? = null, + + open var name: String? = null, +) diff --git a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql index 2146213bea..9542bf4655 100644 --- a/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql +++ b/e2e-tests/spring-rest-multidb/src/main/resources/sql/base.sql @@ -1,4 +1,6 @@ -CREATE TABLE IF NOT EXISTS BaseEntity( +CREATE SCHEMA IF NOT EXISTS foo; + +CREATE TABLE IF NOT EXISTS foo.BaseTable( id VARCHAR(50) NOT NULL, name VARCHAR(50) NOT NULL, constraint pk_x primary key (id) diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt index b5406e5a48..da6aaf88c8 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -1,6 +1,5 @@ package org.evomaster.e2etests.spring.multidb -import org.evomaster.client.java.controller.EmbeddedSutController import org.evomaster.client.java.controller.InstrumentedSutStarter import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType import org.evomaster.client.java.instrumentation.InputProperties @@ -14,6 +13,7 @@ import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource + abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ companion object { From 258a63942cb22a94a55086f06d81707e31be507c Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 13:06:51 +0100 Subject: [PATCH 04/50] more refactoring --- .../java/controller/api/dto/SqlDtoUtils.java | 116 ++++++++++++++++++ .../client/java/sql/DbInfoExtractor.java | 24 ++-- .../client/java/sql/SqlDtoUtils.java | 50 -------- .../sql/internal/QueryResultTransformer.java | 7 +- .../java/sql/internal/SqlNameContext.java | 7 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 22 ++-- 6 files changed, 147 insertions(+), 79 deletions(-) create mode 100644 client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java delete mode 100644 client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java new file mode 100644 index 0000000000..fad84fbf8c --- /dev/null +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java @@ -0,0 +1,116 @@ +package org.evomaster.client.java.controller.api.dto; + +import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; +import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; +import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class SqlDtoUtils { + + + /** + * Return a fully qualifying name for input table, which can be used as id. + */ + public static String getId(TableDto dto){ + if(dto.schema == null){ + return dto.name; + } + return dto.schema + "." + dto.name; + } + + /** + * @return a table DTO for a particular table name + */ + public static TableDto getTable(DbInfoDto schema, String tableName) { + return schema.tables.stream() + .filter(t -> matchByName(t, tableName)) + .findFirst().orElse(null); + } + + /** + * Check if given table dto is matched by name. + * This can be a partial name, or a full qualifying name. + */ + public static boolean matchByName(TableDto dto, String name){ + + Objects.requireNonNull(dto); + Objects.requireNonNull(name); + + if(name.isEmpty()){ + throw new IllegalArgumentException("Empty table name"); + } + + String[] tokens = name.split("\\."); + if(tokens.length > 3){ + throw new IllegalArgumentException("Invalid table name identifier. Too many '.': " + name); + } + if(tokens.length == 1){ + return dto.name.equalsIgnoreCase(tokens[0]); + } + if(tokens.length == 2){ + boolean mn = dto.name.equalsIgnoreCase(tokens[1]); + if(!mn){ + return false; + } + + if(dto.catalog == null && dto.schema == null){ + //there is no default schema, but DTO is unspecified, then false + return tokens[0].equalsIgnoreCase("public"); + } else if(dto.catalog != null && dto.schema != null){ + //both specified... so look at schema + return tokens[0].equalsIgnoreCase(dto.schema); + } else { + //only one specified, take it + if(dto.schema != null){ + return tokens[0].equalsIgnoreCase(dto.schema); + } else{ + //this can be the case for MySQL + return tokens[0].equalsIgnoreCase(dto.catalog); + } + } + } + if(tokens.length == 3){ + return tokens[0].equalsIgnoreCase(dto.catalog) + && tokens[1].equalsIgnoreCase(dto.schema) + && tokens[2].equalsIgnoreCase(dto.name); + } + + //shouldn't be reached + return false; + } + + /** + * + * @param filter specifies which column should be returned, null means all columns should be returned + * @return name of columns based on specified filter + */ + public static List extractColumnNames(InsertionDto dto, Set filter){ + return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.variableName).collect(Collectors.toList()); + } + + /** + * + * @param filter specifies which column should be returned, null means all columns should be returned + * @return printable value of columns based on specified filter + */ + public static List extractColumnPrintableValues(InsertionDto dto, Set filter){ + return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.printableValue).collect(Collectors.toList()); + } + + + /** + * + * @param columnName specified which ColumnDto should be returned based on its name + * @return ColumnDto based on specified columnName + */ + public static ColumnDto extractColumnInfo(TableDto dto, String columnName){ + Optional op = dto.columns.stream().filter(c-> columnName.equalsIgnoreCase(c.name)).findAny(); + return op.orElse(null); + } +} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 9186fa3888..80c959ebfd 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -1,5 +1,6 @@ package org.evomaster.client.java.sql; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import org.evomaster.client.java.controller.api.dto.database.schema.*; import org.evomaster.client.java.sql.internal.constraint.*; import org.evomaster.client.java.utils.SimpleLogger; @@ -42,7 +43,7 @@ private static void checkEnumeratedTypeIsDefined(DbInfoDto schema, TableDto tabl if (schema.enumeraredTypes.stream().noneMatch(k -> k.name.equals(column.type))) { throw new IllegalArgumentException("Missing enumerated type declaration for type " + column.type + " in column " + column.name - + " of table " + table.name); + + " of table " + SqlDtoUtils.getId(table)); } } } @@ -62,9 +63,8 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl } //TODO proper handling of multi-column PKs/FKs - Optional targetTable = schema.tables.stream() - .filter(t -> t.name.equals(fk.get().targetTable)) + .filter(t -> SqlDtoUtils.matchByName(t, fk.get().targetTable)) .findFirst(); if (!targetTable.isPresent()) { @@ -107,7 +107,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl //TODO proper handling of multi-column PKs/FKs Optional targetTable = schema.tables.stream() - .filter(t -> t.name.equals(fk.get().targetTable)) + .filter(t -> SqlDtoUtils.matchByName(t, fk.get().targetTable)) .findFirst(); if (!targetTable.isPresent()) { @@ -235,7 +235,7 @@ private static void addColumnAttributes(DbInfoDto schemaDto, List t.name.equals(tableName.toLowerCase())) + .filter(t -> SqlDtoUtils.matchByName(t,tableName)) .findFirst() .orElse(null); return tableDto.columns.stream() @@ -521,7 +521,9 @@ private static void addConstraints(Connection connection, DatabaseType dt, DbInf private static void addConstraints(DbInfoDto schemaDto, List constraintList) { for (DbTableConstraint constraint : constraintList) { String tableName = constraint.getTableName(); - TableDto tableDto = schemaDto.tables.stream().filter(t -> t.name.equalsIgnoreCase(tableName)).findFirst().orElse(null); + TableDto tableDto = schemaDto.tables.stream() + .filter(t -> SqlDtoUtils.matchByName(t,tableName)) + .findFirst().orElse(null); if (tableDto == null) { throw new NullPointerException("TableDto for table " + tableName + " was not found in the schemaDto"); @@ -808,14 +810,6 @@ private static void addForeignKeyToAutoIncrement(DbInfoDto schema) { } } - /** - * @return a table DTO for a particular table name - */ - private static TableDto getTable(DbInfoDto schema, String tableName) { - return schema.tables.stream() - .filter(t -> t.name.equalsIgnoreCase(tableName)) - .findFirst().orElse(null); - } private static ColumnDto getColumn(TableDto table, String columnName) { return table.columns.stream() @@ -861,7 +855,7 @@ private static boolean isFKToAutoIncrementColumn(DbInfoDto schema, TableDto tabl support for multi-column PKs/FKs */ int positionInFKSequence = fk.sourceColumns.indexOf(columnName); - TableDto targetTableDto = getTable(schema, fk.targetTable); + TableDto targetTableDto = SqlDtoUtils.getTable(schema, fk.targetTable); String targetColumnName = targetTableDto.primaryKeySequence.get(positionInFKSequence); ColumnDto targetColumnDto = getColumn(targetTableDto, targetColumnName); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java deleted file mode 100644 index 0f22c237dd..0000000000 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/SqlDtoUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.evomaster.client.java.sql; - -import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; -import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; -import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -public class SqlDtoUtils { - - - public static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; - } - return dto.schema + "." + dto.name; - } - - /** - * - * @param filter specifies which column should be returned, null means all columns should be returned - * @return name of columns based on specified filter - */ - public static List extractColumnNames(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.variableName).collect(Collectors.toList()); - } - - /** - * - * @param filter specifies which column should be returned, null means all columns should be returned - * @return printable value of columns based on specified filter - */ - public static List extractColumnPrintableValues(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f))).map(i-> i.printableValue).collect(Collectors.toList()); - } - - - /** - * - * @param columnName specified which ColumnDto should be returned based on its name - * @return ColumnDto based on specified columnName - */ - public static ColumnDto extractColumnInfo(TableDto dto, String columnName){ - Optional op = dto.columns.stream().filter(c-> columnName.equalsIgnoreCase(c.name)).findAny(); - return op.orElse(null); - } -} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 47f132a18b..86ce31bb65 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -6,7 +6,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; -import org.evomaster.client.java.sql.SqlDtoUtils; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import org.evomaster.client.java.sql.VariableDescriptor; import java.time.Instant; @@ -140,7 +140,10 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti if (found == null) qr = new QueryResult(relatedColumnNames, tableName); - Optional foundTableSchema = dto.tables.stream().filter(t-> t.name.equalsIgnoreCase(tableName)).findFirst(); + Optional foundTableSchema = dto.tables.stream() + .filter(t-> SqlDtoUtils.matchByName(t,tableName)) + .findFirst(); + if (foundTableSchema.isPresent()){ TableDto tableDto = foundTableSchema.get(); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java index 5ee420f9ce..fb720ead8c 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java @@ -9,6 +9,7 @@ import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.statement.update.Update; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import java.util.*; @@ -67,8 +68,8 @@ public void setSchema(DbInfoDto schema) { * * If no schema is defined, this method returns false. */ - public boolean hasColumn(String tableName, String columnName){ - Objects.requireNonNull(tableName); + public boolean hasColumn(String fullyQualifyingTableName, String columnName){ + Objects.requireNonNull(fullyQualifyingTableName); Objects.requireNonNull(columnName); if(schema == null){ @@ -76,7 +77,7 @@ public boolean hasColumn(String tableName, String columnName){ } return this.schema.tables.stream() - .filter(t -> t.name.equalsIgnoreCase(tableName)) + .filter(t -> SqlDtoUtils.matchByName(t,fullyQualifyingTableName)) .flatMap(t -> t.columns.stream()) .filter(c -> c.name.equalsIgnoreCase(columnName)) .count() > 0; diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 8cf1a5e098..112df7c7cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -1,5 +1,6 @@ package org.evomaster.core.sql +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils import org.evomaster.client.java.controller.api.dto.database.operations.DataRowDto import org.evomaster.client.java.controller.api.dto.database.operations.DatabaseCommandDto import org.evomaster.client.java.controller.api.dto.database.operations.QueryResultDto @@ -96,24 +97,27 @@ class SqlInsertBuilder( val tableConstraints = parseTableConstraints(tableDto).toMutableList() val columns = generateColumnsFrom(tableDto, tableConstraints, schemaDto) - tableToConstraints[tableDto.name] = tableConstraints.toSet() - tableToColumns[tableDto.name] = columns + tableToConstraints[SqlDtoUtils.getId(tableDto)] = tableConstraints.toSet() + tableToColumns[SqlDtoUtils.getId(tableDto)] = columns } // After all columns are loaded, we can load foreign keys for (tableDto in schemaDto.tables) { - tableToForeignKeys[tableDto.name] = calculateForeignKeysFrom(tableDto, tableToColumns) + tableToForeignKeys[SqlDtoUtils.getId(tableDto)] = calculateForeignKeysFrom(tableDto, tableToColumns) } + + // Now we can create the tables for (tableDto in schemaDto.tables) { + val id = SqlDtoUtils.getId(tableDto) val table = Table( - tableDto.name, - tableToColumns[tableDto.name]!!, - tableToForeignKeys[tableDto.name]!!, - tableToConstraints[tableDto.name]!! + id, + tableToColumns[id]!!, + tableToForeignKeys[id]!!, + tableToConstraints[id]!! ) - tables[tableDto.name] = table + tables[id] = table } // Setup extended tables @@ -186,7 +190,7 @@ class SqlInsertBuilder( // val c = targetTable.find { it.name.equals(cname, ignoreCase = true) } // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") - val c = tableToColumns[tableDto.name]!!.find { it.name.equals(cname, ignoreCase = true) } + val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.name} does not have a column called $cname") sourceColumns.add(c) From 949f2689817dac53639d42db4a2fb9e44dc96312 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 14:36:54 +0100 Subject: [PATCH 05/50] moe fixes --- .../controller/internal/SutController.java | 16 +++----- .../evomaster/client/java/sql/DbCleaner.java | 38 +++++++++++++++---- .../client/java/sql/QueryResult.java | 9 ++++- .../core/output/service/TestSuiteWriter.kt | 9 +++-- .../rest/examples/multidb/base/BaseRest.kt | 5 ++- 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 3df05b9e11..39d35d1ddd 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -651,12 +651,6 @@ private boolean registerInitSqlCommands(Connection connection, DbSpecification d return false; } - private void cleanDataInDbConnection(Connection connection, DbSpecification dbSpecification){ - if (dbSpecification.schemaNames != null && !dbSpecification.schemaNames.isEmpty()){ - dbSpecification.schemaNames.forEach(sch-> DbCleaner.clearDatabase(connection, sch, null, dbSpecification.dbType)); - }else - DbCleaner.clearDatabase(connection, null, dbSpecification.dbType); - } /** * Extra information about the SQL Database Schema, if any is present. @@ -1539,10 +1533,12 @@ public void resetDatabase(List tablesToClean) { if (tablesToClean.isEmpty()) return; - if (spec.schemaNames == null || spec.schemaNames.isEmpty()) - DbCleaner.clearDatabase(spec.connection, null, null, tablesToClean, spec.dbType); - else - spec.schemaNames.forEach(sp-> DbCleaner.clearDatabase(spec.connection, sp, null, tablesToClean, spec.dbType)); + DbCleaner.clearTables(spec.connection, tablesToClean, spec.dbType); + +// if (spec.schemaNames == null || spec.schemaNames.isEmpty()) +// DbCleaner.clearDatabase(spec.connection, null, null, tablesToClean, spec.dbType); +// else +// spec.schemaNames.forEach(sp-> DbCleaner.clearDatabase(spec.connection, sp, null, tablesToClean, spec.dbType)); try { handleInitSqlInDbClean(tablesToClean, spec); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index 433eacbb53..a5f4e0b838 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -8,9 +8,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @@ -120,6 +118,36 @@ private static void clearDatabase(int retries, } } + public static void clearTables(Connection connection, List tablesToClean, DatabaseType type) { + + String defaultSchema = getDefaultSchema(type); + // schema -> table_base_name + Map> schemaToNames = new HashMap<>(); + + for(String t : tablesToClean) { + if(! t.contains(".")){ + Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); + x.add(t); + } else { + String[] tokens = t.split("\\."); + String name = tokens[tokens.length - 1]; + String schema = tokens[tokens.length - 2]; + Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); + x.add(name); + } + } + + for(Map.Entry> e : schemaToNames.entrySet()) { + List names = new ArrayList<>(e.getValue()); + clearDatabase(connection, e.getKey(), null, names, type, true); + } + } + + + public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { + clearDatabase(connection, tableToSkip, tableToClean, type, true); + } + public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type, boolean doResetSequence) { clearDatabase(connection, getDefaultSchema(type), tablesToSkip, type, doResetSequence); } @@ -128,10 +156,6 @@ public static void clearDatabase(Connection connection, List tablesToSki clearDatabase(connection, tablesToSkip, type, true); } - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, tableToSkip, tableToClean, type, true); - } - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { clearDatabase(connection, getDefaultSchema(type), tableToSkip, tableToClean, type, doResetSequence); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java index ad11f36de8..c900b476d7 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResult.java @@ -68,10 +68,17 @@ public QueryResult(ResultSet resultSet) { for (int i = 0; i < md.getColumnCount(); i++) { int index = i + 1; + String schema = md.getSchemaName(index); + String name; + if (schema != null) { + name = schema + "." + md.getTableName(index); + } else { + name = md.getTableName(index); + } VariableDescriptor desc = new VariableDescriptor( getColumnName(md, index), md.getColumnLabel(index), - md.getTableName(index) + name ); variableDescriptors.add(desc); } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 5e30dc444b..4753b28ae1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -1,6 +1,7 @@ package org.evomaster.core.output.service import com.google.inject.Inject +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto import org.evomaster.client.java.controller.api.dto.database.operations.MongoInsertionDto import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils @@ -207,9 +208,11 @@ class TestSuiteWriter { } val all = sampler.extractFkTables(accessedTable) - //if (all.isEmpty()) return "null" - - val tableNamesInSchema = remoteController.getCachedSutInfo()?.sqlSchemaDto?.tables?.map { it.name }?.toSet() + val tableNamesInSchema = remoteController.getCachedSutInfo() + ?.sqlSchemaDto + ?.tables + ?.map { SqlDtoUtils.getId(it) } + ?.toSet() ?: setOf() val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y.equals(x,true) } }.sorted() diff --git a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt index 6f35333041..f2298d4971 100644 --- a/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt +++ b/e2e-tests/spring-rest-multidb/src/main/kotlin/com/foo/rest/examples/multidb/base/BaseRest.kt @@ -1,6 +1,7 @@ package com.foo.rest.examples.multidb.base import org.springframework.http.ResponseEntity +import org.springframework.jdbc.core.BeanPropertyRowMapper import org.springframework.jdbc.core.JdbcTemplate import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -17,9 +18,9 @@ open class BaseRest( open fun get(@PathVariable id: String) : ResponseEntity { - val x = jdbc.queryForObject("SELECT * FROM foo.BaseTable WHERE id = '$id'", BaseTable::class.java) + val x = jdbc.query("SELECT * FROM foo.BaseTable WHERE id = '$id'", BeanPropertyRowMapper(BaseTable::class.java)) - if(x != null) { + if(x.isNotEmpty()) { return ResponseEntity.ok("OK") } From effb4c0b9eaa0b63e752f1dc996739c3a1800156 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 14:46:46 +0100 Subject: [PATCH 06/50] refactoring package location --- .../client/java/sql/{ => cleaner}/DbCleanerH2Test.java | 4 +++- .../java/sql/{ => cleaner}/DbCleanerMariaDBTest.java | 3 ++- .../client/java/sql/{ => cleaner}/DbCleanerMySQLTest.java | 3 ++- .../java/sql/{ => cleaner}/DbCleanerPostgresTest.java | 4 +++- .../java/sql/{ => cleaner}/DbCleanerSQLServerTest.java | 8 +++++++- .../{ => cleaner}/DbCleanerSQLServerWithSchemaTest.java | 6 +++++- .../client/java/sql/{ => cleaner}/DbCleanerTestBase.java | 2 +- 7 files changed, 23 insertions(+), 7 deletions(-) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerH2Test.java (89%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerMariaDBTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerMySQLTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerPostgresTest.java (93%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerSQLServerTest.java (93%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerSQLServerWithSchemaTest.java (95%) rename client-java/sql/src/test/java/org/evomaster/client/java/sql/{ => cleaner}/DbCleanerTestBase.java (99%) diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java similarity index 89% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java index 0f45cf1f73..86d331d8e4 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerH2Test.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java @@ -1,6 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java index c1155071e3..9a046be9a3 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMariaDBTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java @@ -1,6 +1,7 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java index b64ab9a78b..6e63975ff9 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerMySQLTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java @@ -1,7 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java similarity index 93% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java index 0737dd04e5..2cb69332a7 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerPostgresTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java @@ -1,6 +1,8 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java similarity index 93% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java index c32b1cb8d4..a452a91600 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java @@ -1,12 +1,15 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; //TODO //import org.evomaster.e2etests.utils.CIUtils; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.client.java.sql.DbCleaner; +import org.evomaster.client.java.sql.SqlScriptRunner; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -17,6 +20,9 @@ import java.util.HashMap; import java.util.List; +//No longer support MSSQL +@Deprecated +@Disabled public class DbCleanerSQLServerTest extends DbCleanerTestBase{ private static final int PORT = 1433; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java similarity index 95% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java index e9e40b8d69..0dd838c7d0 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerSQLServerWithSchemaTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java @@ -1,10 +1,11 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.ci.utils.CIUtils; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; import org.evomaster.client.java.sql.DbCleaner; import org.evomaster.client.java.sql.QueryResult; import org.evomaster.client.java.sql.SqlScriptRunner; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; @@ -16,6 +17,9 @@ import static org.junit.jupiter.api.Assertions.*; +//No longer support MSSQL +@Deprecated +@Disabled public class DbCleanerSQLServerWithSchemaTest { private static final int PORT = 1433; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java similarity index 99% rename from client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java rename to client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java index bfeafb0983..b819fb32e6 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/DbCleanerTestBase.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerTestBase.java @@ -1,4 +1,4 @@ -package org.evomaster.client.java.sql; +package org.evomaster.client.java.sql.cleaner; import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType; import org.evomaster.client.java.sql.QueryResult; From 506e4174478632313e37dab2c552c370b0bbc149 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 15:54:18 +0100 Subject: [PATCH 07/50] more cleaning and refactoring for DbCleaner --- .../controller/internal/SutController.java | 6 +- .../sql/h2/DatabaseFakeH2SutController.java | 2 +- .../mysql/DatabaseFakeMySQLSutController.java | 2 +- .../evomaster/client/java/sql/DbCleaner.java | 195 ++++++++---------- .../java/sql/cleaner/DbCleanerH2Test.java | 2 +- .../sql/cleaner/DbCleanerMariaDBTest.java | 2 +- .../java/sql/cleaner/DbCleanerMySQLTest.java | 2 +- .../sql/cleaner/DbCleanerPostgresTest.java | 2 +- .../sql/cleaner/DbCleanerSQLServerTest.java | 2 +- .../DbCleanerSQLServerWithSchemaTest.java | 4 +- .../spring/db/auth/DbAuthController.java | 2 +- .../MultiDbParameterizedE2ETemplate.kt | 3 + .../spring/db/auth/DbAuthController.java | 2 +- 13 files changed, 103 insertions(+), 123 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 39d35d1ddd..4924466a7e 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -499,9 +499,9 @@ public final void cleanAccessedTables(){ List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ - emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); + emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType, true)); } else { - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); + DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType, true); } tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); } @@ -1522,7 +1522,7 @@ public void resetDatabase(List tablesToClean) { if(tablesToClean == null){ // all data will be reset - DbCleaner.clearDatabase(spec.connection, null, null, null, spec.dbType); + DbCleaner.clearDatabase(spec.connection, null, null, null, spec.dbType, true); try { reAddAllInitSql(); } catch (SQLException e) { diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java index ea0b417ace..400b41de2d 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/DatabaseFakeH2SutController.java @@ -52,7 +52,7 @@ public SutInfoDto.OutputFormat getPreferredOutputFormat() { @Override public String startSut() { running = true; - DbCleaner.clearDatabase(sqlConnection, null, DatabaseType.H2); + DbCleaner.clearDatabase_H2(sqlConnection, null, null); return "foo"; } diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java index 1f47e54271..33a149ca7c 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/DatabaseFakeMySQLSutController.java @@ -55,7 +55,7 @@ public SutInfoDto.OutputFormat getPreferredOutputFormat() { public String startSut() { running = true; - DbCleaner.clearDatabase(sqlConnection, DB_NAME,null, DatabaseType.MYSQL); + DbCleaner.clearDatabase_MySQL(sqlConnection, DB_NAME,null, null); return "foo"; } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index a5f4e0b838..007233ee34 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -17,19 +17,38 @@ */ public class DbCleaner { - public static void clearDatabase_H2(Connection connection) { - clearDatabase_H2(connection, null); - } - public static void clearDatabase_H2(Connection connection, List tableToSkip) { - clearDatabase_H2(connection, getDefaultSchema(DatabaseType.H2), tableToSkip); + public static void clearTables(Connection connection, List fullyQualifyingTableNames, DatabaseType type) { + + String defaultSchema = getDefaultSchema(type); + // schema -> table_base_name + Map> schemaToNames = new HashMap<>(); + + for(String t : fullyQualifyingTableNames) { + if(! t.contains(".")){ + Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); + x.add(t); + } else { + String[] tokens = t.split("\\."); + String name = tokens[tokens.length - 1]; + String schema = tokens[tokens.length - 2]; + Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); + x.add(name); + } + } + + for(Map.Entry> e : schemaToNames.entrySet()) { + List names = new ArrayList<>(e.getValue()); + clearDatabase(connection, e.getKey(), null, names, type, true); + } } - public static void clearDatabase_H2(Connection connection, String schemaName, List tableToSkip) { - clearDatabase_H2(connection, schemaName, tableToSkip, null); + public static void clearDatabase_Postgres(Connection connection, String schemaName, List tableToSkip, List tableToClean) { + clearDatabase(getDefaultRetries(DatabaseType.POSTGRES), connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tableToSkip, tableToClean, DatabaseType.POSTGRES, false, true, false); } - public static void clearDatabase_H2(Connection connection, String schemaName, List tableToSkip, List tableToClean) { + public static void clearDatabase_H2(Connection connection, List tableToSkip, List tableToClean) { + String schemaName = getDefaultSchema(DatabaseType.H2); final String h2Version; try { h2Version = H2VersionUtils.getH2Version(connection); @@ -40,14 +59,63 @@ public static void clearDatabase_H2(Connection connection, String schemaName, Li * The SQL command "TRUNCATE TABLE my_table RESTART IDENTITY" * is not supported by H2 version 1.4.199 or lower */ - final boolean restartIdentitiyWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); + boolean restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); clearDatabase(getDefaultRetries(DatabaseType.H2), connection, schemaName, tableToSkip, tableToClean, DatabaseType.H2, - false, true, restartIdentitiyWhenTruncating); + false, true, restartIdentityWhenTruncating); + } + + + public static void clearDatabase_MySQL(Connection connection, String schemaName, List tablesToSkip, List tableToClean) { + clearDatabase(connection, getSchemaName(schemaName, DatabaseType.MYSQL), tablesToSkip, tableToClean, DatabaseType.MYSQL, true); } + //TODO why this function? + public static void dropDatabaseTables(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { + if (type != DatabaseType.MYSQL && type != DatabaseType.MARIADB) + throw new IllegalArgumentException("Dropping tables are not supported by " + type); + clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tablesToSkip, null, type, true, true, false); + } + + + public static void clearDatabase( + Connection connection, + String schemaName, + List tableToSkip, + List tableToClean, + DatabaseType type, + boolean doResetSequence + ) { + /* + * Enable the restarting of Identity fields only if sequences are to be restarted + * and the database type is H2 + */ + boolean restartIdentityWhenTruncating; + if (doResetSequence && type.equals(DatabaseType.H2)) { + try { + String h2Version = H2VersionUtils.getH2Version(connection); + restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); + } catch (SQLException ex) { + throw new RuntimeException("Unexpected SQL exception while getting H2 version", ex); + } + } else { + restartIdentityWhenTruncating = false; + } + clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, false, doResetSequence, restartIdentityWhenTruncating); + } + + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + private static String getSchemaName(String schemaName, DatabaseType type) { + if (schemaName != null) return schemaName; + return getDefaultSchema(type); + } + + /* - [non-determinism-source] Man: retries might lead to non-determinate logs - */ + [non-determinism-source] Man: retries might lead to non-determinate logs + */ private static void clearDatabase(int retries, Connection connection, String schemaName, @@ -118,98 +186,6 @@ private static void clearDatabase(int retries, } } - public static void clearTables(Connection connection, List tablesToClean, DatabaseType type) { - - String defaultSchema = getDefaultSchema(type); - // schema -> table_base_name - Map> schemaToNames = new HashMap<>(); - - for(String t : tablesToClean) { - if(! t.contains(".")){ - Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); - x.add(t); - } else { - String[] tokens = t.split("\\."); - String name = tokens[tokens.length - 1]; - String schema = tokens[tokens.length - 2]; - Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); - x.add(name); - } - } - - for(Map.Entry> e : schemaToNames.entrySet()) { - List names = new ArrayList<>(e.getValue()); - clearDatabase(connection, e.getKey(), null, names, type, true); - } - } - - - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, tableToSkip, tableToClean, type, true); - } - - public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getDefaultSchema(type), tablesToSkip, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, List tablesToSkip, DatabaseType type) { - clearDatabase(connection, tablesToSkip, type, true); - } - - public static void clearDatabase(Connection connection, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getDefaultSchema(type), tableToSkip, tableToClean, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { - clearDatabase(connection, getSchemaName(schemaName, type), tablesToSkip, type, true); - } - - public static void clearDatabase(Connection connection, String schemaName, List tablesToSkip, DatabaseType type, boolean doResetSequence) { - clearDatabase(connection, getSchemaName(schemaName, type), tablesToSkip, null, type, doResetSequence); - } - - public static void clearDatabase(Connection connection, String schemaName, List tableToSkip, List tableToClean, DatabaseType type) { - clearDatabase(connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, true); - } - - public static void clearDatabase(Connection connection, String schemaName, List tableToSkip, List tableToClean, DatabaseType type, boolean doResetSequence) { - /* - * Enable the restarting of Identity fields only if sequences are to be restarted - * and the database type is H2 - */ - boolean restartIdentityWhenTruncating; - if (doResetSequence && type.equals(DatabaseType.H2)) { - try { - String h2Version = H2VersionUtils.getH2Version(connection); - restartIdentityWhenTruncating = H2VersionUtils.isVersionGreaterOrEqual(h2Version, H2VersionUtils.H2_VERSION_2_0_0); - } catch (SQLException ex) { - throw new RuntimeException("Unexpected SQL exception while getting H2 version", ex); - } - } else { - restartIdentityWhenTruncating = false; - } - clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tableToSkip, tableToClean, type, false, doResetSequence, restartIdentityWhenTruncating); - } - - public static void dropDatabaseTables(Connection connection, String schemaName, List tablesToSkip, DatabaseType type) { - if (type != DatabaseType.MYSQL && type != DatabaseType.MARIADB) - throw new IllegalArgumentException("Dropping tables are not supported by " + type); - clearDatabase(getDefaultRetries(type), connection, getSchemaName(schemaName, type), tablesToSkip, null, type, true, true, false); - } - - - public static void clearDatabase_Postgres(Connection connection, String schemaName, List tablesToSkip) { - clearDatabase_Postgres(connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tablesToSkip, null); - } - - public static void clearDatabase_Postgres(Connection connection, String schemaName, List tableToSkip, List tableToClean) { - clearDatabase(getDefaultRetries(DatabaseType.POSTGRES), connection, getSchemaName(schemaName, DatabaseType.POSTGRES), tableToSkip, tableToClean, DatabaseType.POSTGRES, false, true, false); - } - - private static String getSchemaName(String schemaName, DatabaseType type) { - if (schemaName != null) return schemaName; - return getDefaultSchema(type); - } /** * @param tableToSkip are tables to skip @@ -228,9 +204,12 @@ private static List cleanDataInTables(List tableToSkip, String schema, boolean singleCommand, boolean doDropTable, - boolean restartIdentityWhenTruncating) throws SQLException { - if (tableToSkip != null && (!tableToSkip.isEmpty()) && tableToClean != null && (!tableToClean.isEmpty())) + boolean restartIdentityWhenTruncating + ) throws SQLException { + + if (tableToSkip != null && (!tableToSkip.isEmpty()) && tableToClean != null && (!tableToClean.isEmpty())) { throw new IllegalArgumentException("tableToSkip and tableToClean cannot be configured at the same time."); + } // Find all tables and truncate them Set tables = new HashSet<>(); @@ -331,7 +310,7 @@ private static void deleteTables(Statement statement, String table, String schem if (!schema.isEmpty() && !schema.equals(getDefaultSchema(DatabaseType.MS_SQL_SERVER))) tableWithSchema = schema + "." + schema; statement.executeUpdate("DELETE FROM " + tableWithSchema); -// NOTE TAHT ideally we should reseed identify here, but there would case an issue, i.e., does not contain an identity column +// NOTE THAT ideally we should reseed identify here, but there would case an issue, i.e., does not contain an identity column if (tableHasIdentify.contains(table)) statement.executeUpdate("DBCC CHECKIDENT ('" + tableWithSchema + "', RESEED, 0)"); } @@ -468,8 +447,6 @@ private static String getAllTableCommand(DatabaseType type, String schema) { return command + " AND TABLE_SCHEMA='" + schema + "'"; } throw new DbUnsupportedException(type); -// case MS_SQL_SERVER_2000: -// return "SELECT sobjects.name FROM sysobjects sobjects WHERE sobjects.xtype = '"+schema+"'"; // shcema can be 'U' } private static String getAllSequenceCommand(DatabaseType type, String schemaName) { diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java index 86d331d8e4..05486fd36f 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerH2Test.java @@ -44,7 +44,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase_H2(connection, "PUBLIC", tablesToSkip, tableToClean); + DbCleaner.clearDatabase_H2(connection, tablesToSkip, tableToClean); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java index 9a046be9a3..d4da1809bc 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMariaDBTest.java @@ -59,7 +59,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, getDbType()); + DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, getDbType(), true); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java index 6e63975ff9..135d271116 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerMySQLTest.java @@ -65,7 +65,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, DatabaseType.MYSQL); + DbCleaner.clearDatabase(connection, DB_NAME, tablesToSkip, tableToClean, DatabaseType.MYSQL, true); } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java index 2cb69332a7..866741ec7d 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerPostgresTest.java @@ -38,7 +38,7 @@ public static void initClass() throws Exception{ } @AfterAll - private static void afterClass() throws Exception{ + public static void afterClass() throws Exception{ connection.close(); postgres.stop(); } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java index a452a91600..7ae17be264 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerTest.java @@ -94,7 +94,7 @@ protected Connection getConnection() { @Override protected void clearDatabase(List tablesToSkip, List tableToClean) { - DbCleaner.clearDatabase(connection, tablesToSkip, tableToClean, getDbType()); + //DbCleaner.clearDatabase(connection, tablesToSkip, tableToClean, getDbType()); } @Override diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java index 0dd838c7d0..e84ea99ac7 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/cleaner/DbCleanerSQLServerWithSchemaTest.java @@ -63,10 +63,10 @@ public void testSkipTableMisconfigured() throws Exception{ assertThrows(RuntimeException.class, ()-> - DbCleaner.clearDatabase(connection, "", null, null, DatabaseType.MS_SQL_SERVER) + DbCleaner.clearDatabase(connection, "", null, null, DatabaseType.MS_SQL_SERVER, true) ); - DbCleaner.clearDatabase(connection, "Foo", null, null, DatabaseType.MS_SQL_SERVER); + DbCleaner.clearDatabase(connection, "Foo", null, null, DatabaseType.MS_SQL_SERVER, true); res = SqlScriptRunner.execCommand(connection, "SELECT * FROM Foo.Foo;"); assertEquals(0, res.seeRows().size()); } diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java b/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java index d38c8619ae..706cd6b268 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java @@ -29,7 +29,7 @@ public List getInfoForAuthentication() { @Override public void resetStateOfSUT() { - DbCleaner.clearDatabase_H2(sqlConnection); + DbCleaner.clearDatabase_H2(sqlConnection,null,null); try { SqlScriptRunner.execInsert(sqlConnection, diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt index da6aaf88c8..7ab653833f 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -10,10 +10,13 @@ import org.evomaster.e2etests.utils.EnterpriseTestBase import org.evomaster.e2etests.utils.RestTestBase import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Disabled import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource +//FIXME put back once fixed +@Disabled abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ companion object { diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java index c34fbb21dc..cf05930fa7 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/com/foo/rest/examples/spring/db/auth/DbAuthController.java @@ -29,7 +29,7 @@ public List getInfoForAuthentication() { @Override public void resetStateOfSUT() { - DbCleaner.clearDatabase_H2(sqlConnection); + DbCleaner.clearDatabase_H2(sqlConnection, null, null); try { SqlScriptRunner.execInsert(sqlConnection, From fa9fee10136e4139cd56962643607f5dfeac7743 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 16 Dec 2024 22:22:42 +0100 Subject: [PATCH 08/50] fix for failing test --- .../evomaster/client/java/sql/DataRow.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java index 991d2c16f4..541940a4c5 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java @@ -98,23 +98,41 @@ public Object getValueByName(String name, String table) { } } + List candidates = new ArrayList<>(); + //if none, then check column names for (int i = 0; i < variableDescriptors.size(); i++) { VariableDescriptor desc = variableDescriptors.get(i); - if (n.equalsIgnoreCase(desc.getColumnName()) && - (t == null || t.isEmpty() - || t.equalsIgnoreCase(desc.getTableName()) - /* - TODO: this does not cover all possible cases, as in theory - there can be many unnamed tables (eg results of sub-selects) - with same column names. At this moment, we would not - be able to distinguish them - */ - || t.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE) - ) - ) { + + if (!n.equalsIgnoreCase(desc.getColumnName())){ + continue; + } + //no defined table, or exact match + if(t == null || t.isEmpty() || t.equalsIgnoreCase(desc.getTableName())){ return getValue(i); } + /* + TODO: this does not cover all possible cases, as in theory + there can be many unnamed tables (eg results of sub-selects) + with same column names. At this moment, we would not + be able to distinguish them + */ + if(t.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)){ + candidates.add(i); + } + /* + We just specified the name without schema... if unique, we would be fine + */ + if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ + candidates.add(i); + } + } + if(candidates.size() > 1){ + SimpleLogger.uniqueWarn("More than one table candidate for: " + t); + } + + if(candidates.size() >= 1){ + return getValue(candidates.get(0)); } throw new IllegalArgumentException("No variable called '" + name + "' for table '" + table + "'"); From 5c5a4e40d151ccfc3c75264c66634200a3e4843c Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 17 Dec 2024 13:28:07 +0100 Subject: [PATCH 09/50] more fixes related to schema in table ids --- .../evomaster/core/sql/SqlInsertBuilder.kt | 66 +++++++++++++++---- .../core/sql/SqlInsertBuilderTest.kt | 6 +- .../sql/extract/h2/CatwatchSqlExtractTest.kt | 4 +- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 112df7c7cf..701dfcdd2c 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -180,9 +180,11 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { - tableToColumns[fk.targetTable] - ?: throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") + val tableKey = getTableKey(tableToColumns.keys, fk.targetTable) + if(tableKey == null || tableToColumns[tableKey] == null) { + throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") + } val sourceColumns = mutableSetOf() for (cname in fk.sourceColumns) { @@ -196,7 +198,7 @@ class SqlInsertBuilder( sourceColumns.add(c) } - fks.add(ForeignKey(sourceColumns, fk.targetTable)) + fks.add(ForeignKey(sourceColumns, tableKey)) } return fks } @@ -616,19 +618,52 @@ class SqlInsertBuilder( /** * SQL is not case sensitivity. * Therefore, table/column must ignore case sensitivity. + * + * No. This is not strictly true: + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * + * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = tables.keys.any { it.equals(tableName, ignoreCase = true) } + fun isTable(tableName: String) = getTableKey(tables.keys, tableName) != null - fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val data = if (useExtraConstraints) extendedTables else tables + val key = getTableKey(map.keys, tableName) + return map[key] + } + private fun getTableKey(keys: Set, tableName: String) : String?{ /* * SQL is not case sensitivity, table/column must ignore case sensitivity. + * No, this is not really true... + * Usually, names are lowered-cased by the DB, unless quoted in "": + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * */ - val tableNameKey = data.keys.find { tableName.equals(it, ignoreCase = true) } - return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") + val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + if (!tableName.contains(".") && tableNameKey == null){ + //input name might be without schema, so check for partial match + val candidates = keys.filter { it.endsWith(".${tableName}", true) } + if(candidates.size > 1){ + throw IllegalArgumentException("Ambiguity." + + " More than one candidate of table called $tableName." + + " Values: ${candidates.joinToString(", ")}") + } + if(candidates.size == 1){ + return candidates[0] + } + } + return tableNameKey + } + + fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + + val data = if (useExtraConstraints) extendedTables else tables + + val tableNameKey = getTableKey(data.keys, tableName) + ?: throw IllegalArgumentException("No table called $tableName") + return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") } /** @@ -1009,11 +1044,18 @@ class SqlInsertBuilder( private fun formatNameInSql(name: String): String { + return name + + /* + The handling of "" quotes is extremely tricky. + TODO if we really want to support it, would need to have a flag on each name (eg have + to pass them as pojo, not string) + */ //TODO: why this??? needs explanation - return when { - databaseType == DatabaseType.MYSQL || name == SQLKey.ALL.key -> name - else -> "\"$name\"" - } +// return when { +// databaseType == DatabaseType.MYSQL || name == SQLKey.ALL.key -> name +// else -> "\"$name\"" +// } } /** diff --git a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt index 2bbd4c78d0..ed40fa4094 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt @@ -1067,9 +1067,9 @@ class SqlInsertBuilderTest { // ABCD or ACBD assertEquals(4, enabled.size) - assertEquals("ROOTA", enabled[0].table.name) - assertEquals("LEAFD", enabled[3].table.name) - assertTrue(enabled.subList(1,3).map { it.table.name }.containsAll(listOf("NODEB", "NODEC"))) + assertEquals("PUBLIC.ROOTA", enabled[0].table.name) + assertEquals("PUBLIC.LEAFD", enabled[3].table.name) + assertTrue(enabled.subList(1,3).map { it.table.name }.containsAll(listOf("PUBLIC.NODEB", "PUBLIC.NODEC"))) } ////// Tests Numeric parsing for Min/Max Bounds diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt index d217f57dae..c0ebd13d7c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt @@ -59,8 +59,8 @@ class CatwatchSqlExtractTest : ExtractTestBaseH2(){ SqlActionUtils.randomizeDbActionGenes(insertions.toMutableList(), Randomness()) assertEquals(2, insertions.size) - assert(insertions[0].table.name.equals("PROJECT", ignoreCase = true)) - assert(insertions[1].table.name.equals("lANGUAGE_LIST", ignoreCase = true)) + assert(insertions[0].table.name.equals("PUBLIC.PROJECT", ignoreCase = true)) + assert(insertions[1].table.name.equals("PUBLIC.lANGUAGE_LIST", ignoreCase = true)) val projectId = (insertions[0].seeTopGenes().filterIsInstance()).first().uniqueId From a572f059cb9ca1e291dcdca7515abb4b9f4f66b7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 18 Dec 2024 10:54:25 +0100 Subject: [PATCH 10/50] more cleaning and fixing --- .../problem/rest/resource/ResourceCluster.kt | 10 +- .../dependency/ResourceRelatedToTable.kt | 101 ++++++++++++------ .../org/evomaster/core/sql/SqlActionUtils.kt | 52 ++++++++- .../evomaster/core/sql/SqlInsertBuilder.kt | 32 +----- .../org/evomaster/core/sql/schema/Table.kt | 3 + .../rest/resource/ResourceNodeWithDbTest.kt | 4 +- 6 files changed, 137 insertions(+), 65 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt index 5043156bc2..a2bc2bfe3a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt @@ -127,16 +127,16 @@ class ResourceCluster { * @return existing rows of the table based on the specified [tableName] */ fun getDataInDb(tableName: String) : MutableList?{ - val found = dataInDB.filterKeys { k-> k.equals(tableName, ignoreCase = true) }.keys - if (found.isEmpty()) return null - Lazy.assert{found.size == 1} - return dataInDB.getValue(found.first()) + val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) + ?: return null + return dataInDB.getValue(key) } /** * @return table class based on specified [name] */ - fun getTableByName(name : String) = tables.keys.find { it.equals(name, ignoreCase = true) }?.run { tables[this] } + private fun getTableByName(name : String) = + SqlActionUtils.getTableKey(tables.keys, name)?.run { tables[this] } /** diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index 579494cd37..bc9d513124 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -5,6 +5,7 @@ import org.evomaster.core.sql.SQLKey import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo +import org.evomaster.core.sql.SqlActionUtils /** * related info between resource and tables @@ -58,8 +59,14 @@ class ResourceRelatedToTable(val key: String) { fun updateActionRelatedToTable(verb : String, dto: SqlExecutionsDto, existingTables : Set) : Boolean{ - val tables = mutableListOf().plus(dto.deletedData).plus(dto.updatedData.keys).plus(dto.insertedData.keys).plus(dto.queriedData.keys) - .filter { existingTables.contains(it) || existingTables.any { e->e.toLowerCase() == it.toLowerCase() }}.toHashSet() + val tables = mutableListOf() + .plus(dto.deletedData) + .plus(dto.updatedData.keys) + .plus(dto.insertedData.keys) + .plus(dto.queriedData.keys) + .filter { x -> + existingTables.any { x.equals(it, true) } + }.toHashSet() if (tables.isEmpty()) return false @@ -74,7 +81,7 @@ class ResourceRelatedToTable(val key: String) { actionToTables[verb]!!.add(access) } - access.updateTableWithFields(dto.deletedData.map { Pair(it, mutableSetOf()) }.toMap(), SQLKey.DELETE) + access.updateTableWithFields(dto.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) access.updateTableWithFields(dto.insertedData, SQLKey.INSERT) access.updateTableWithFields(dto.queriedData, SQLKey.SELECT) access.updateTableWithFields(dto.updatedData, SQLKey.UPDATE) @@ -83,15 +90,23 @@ class ResourceRelatedToTable(val key: String) { } fun getConfirmedDirectTables() : Set{ - return derivedMap.keys.filter { t-> confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } }.toHashSet() + return derivedMap.keys + .filter { t-> + confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } + }.toHashSet() } - fun findBestTableForParam(tables: Set, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair, Double>? { - val map = simpleP2Table.derivedMap.filter { tables.any { t-> t.equals(it.key, ignoreCase = true) } } - .map { + fun findBestTableForParam( + tables: Set, + simpleP2Table : SimpleParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair, Double>? { + val map = simpleP2Table.derivedMap + .filter { d -> tables.any { t-> SqlActionUtils.isMatchingTableName(d.key, t) } } + .map { Pair(it.key, it.value.similarity) - }.toMap() + }.toMap() if(map.isEmpty()) return null val best = map.asSequence().sortedBy { it.value }.last().value return Pair(map.filter { it.value == best }.keys, best) @@ -100,11 +115,18 @@ class ResourceRelatedToTable(val key: String) { /** * return Pair.first is name of table, Pair.second is column of Table */ - fun getSimpleParamToSpecifiedTable(table: String, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair? { - simpleP2Table.derivedMap.filter { table.equals(it.key, ignoreCase = true) }.let { map-> - if (map.isEmpty()) return null - else return Pair(table, map.values.first().targetMatched) - } + fun getSimpleParamToSpecifiedTable( + table: String, + simpleP2Table : SimpleParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair? { + + return simpleP2Table.derivedMap + .filter { table.equals(it.key, ignoreCase = true) } + .let { map-> + if (map.isEmpty()) null + else Pair(table, map.values.first().targetMatched) + } } @@ -115,7 +137,9 @@ class ResourceRelatedToTable(val key: String) { */ fun findBestTableForParam(tables: Set, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : MutableMap, Double>>?{ val fmap = bodyP2Table.fieldsMap - .filter { it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } } + .filter { + it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } + } if(fmap.isEmpty()) return null val result : MutableMap, Double>> = mutableMapOf() @@ -134,7 +158,9 @@ class ResourceRelatedToTable(val key: String) { */ fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair>? { val fmap = bodyP2Table.fieldsMap - .filter { it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } } + .filter { + it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } + } if(fmap.isEmpty()) return null else{ return Pair(table, fmap.map { f-> Pair(f.key, f.value.getRelatedColumn(table)!!.first())}.toMap()) @@ -148,7 +174,7 @@ class ResourceRelatedToTable(val key: String) { */ fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, fieldName : String, onlyConfirmedColumn : Boolean = false) : Pair>? { val fmap = bodyP2Table.fieldsMap[fieldName]?: return null - fmap.derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let { + fmap.derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}.let { if (it.isEmpty()) return null else return Pair(table, Pair(fieldName, it.values.first().targetMatched)) } @@ -165,9 +191,10 @@ class ResourceRelatedToTable(val key: String) { } private fun getTablesInDerivedMap(input : String) : List{ - return derivedMap.values.flatMap { - it.filter { m-> m.input.toLowerCase() == input.toLowerCase()} - } + return derivedMap.values + .flatMap { + it.filter { m-> m.input.toLowerCase() == input.toLowerCase()} + } } } @@ -232,7 +259,13 @@ abstract class ParamRelatedToTable ( val confirmedColumn : MutableSet = mutableSetOf() - open fun getRelatedColumn(table: String) : Set? = derivedMap.filterKeys { it.equals(table, ignoreCase = true) }.values.firstOrNull().run { if (this == null) null else setOf(this.targetMatched) } + open fun getRelatedColumn(table: String) : Set? = + derivedMap.filterKeys { SqlActionUtils.isMatchingTableName(it, table)} + .values.firstOrNull() + .run { + if (this == null) null + else setOf(this.targetMatched) + } } /** @@ -254,12 +287,19 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ val fieldsMap : MutableMap = mutableMapOf() - override fun getRelatedColumn(table: String) : Set? = fieldsMap.values.filter { it.derivedMap.any { m-> m.key.toLowerCase() == table.toLowerCase() }}.run { - if (this.isEmpty()) return null - else - this.map { f->f.derivedMap.filterKeys { k->k.toLowerCase() == table.toLowerCase() }.asSequence().first().value.targetMatched}.toHashSet() - } - + override fun getRelatedColumn(table: String) : Set? = + fieldsMap.values + .filter { + it.derivedMap.any { m-> SqlActionUtils.isMatchingTableName(m.key, table)} + }.run { + if (this.isEmpty()) + return null + else + this.map { f-> + f.derivedMap.filterKeys { k -> SqlActionUtils.isMatchingTableName(k, table) } + .asSequence().first().value.targetMatched + }.toHashSet() + } } /** * related info between a field of BodyParam and a table @@ -267,10 +307,11 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT class ParamFieldRelatedToTable(key: String) : ParamRelatedToTable(key){ override fun getRelatedColumn(table: String) : Set? { - derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let { - if(it.isEmpty()) return null - else return it.values.map {m-> m.targetMatched}.toSet() - } + return derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)} + .let { + if(it.isEmpty()) null + else it.values.map {m-> m.targetMatched}.toSet() + } } } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index baca2044ea..10e2504fc7 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -495,7 +495,57 @@ object SqlActionUtils { * @return a list of dbactions from [sqlActions] whose related table is [tableName] */ fun findDbActionsByTableName(sqlActions: List, tableName : String) : List{ - return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) } + return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) + || it.table.name.endsWith(".$tableName", true)} } + + /** + * Are the 2 names matching? This ignore case. + * The first [fullName] is a full qualifying name, including schema. + * The second [name] "might" be simple, or full qualifying. + */ + fun isMatchingTableName(fullName: String, name: String) : Boolean{ + + if(fullName.equals(name, ignoreCase = true)){ + return true + } + + if(name.contains(".")){ + return false + } + + return fullName.endsWith(".$name", true) + } + + /** + * Given a set of table ids, returns the one matching the given table name. + * There are at least 2 problems handled here: + * 1) case sensitiveness + * 2) keys having full names (eg, including schemas and possibly catalog) whereas input only having the name. + * this latter case is not a problem if names are unique + */ + fun getTableKey(keys: Set, tableName: String) : String?{ + /* + * SQL is not case sensitivity, table/column must ignore case sensitivity. + * No, this is not really true... + * Usually, names are lowered-cased by the DB, unless quoted in "": + * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. + * + */ + val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + if (!tableName.contains(".") && tableNameKey == null){ + //input name might be without schema, so check for partial match + val candidates = keys.filter { it.endsWith(".${tableName}", true) } + if(candidates.size > 1){ + throw IllegalArgumentException("Ambiguity." + + " More than one candidate of table called $tableName." + + " Values: ${candidates.joinToString(", ")}") + } + if(candidates.size == 1){ + return candidates[0] + } + } + return tableNameKey + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 701dfcdd2c..bdd522b44a 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -180,7 +180,7 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { - val tableKey = getTableKey(tableToColumns.keys, fk.targetTable) + val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -624,43 +624,21 @@ class SqlInsertBuilder( * * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = getTableKey(tables.keys, tableName) != null + fun isTable(tableName: String) = SqlActionUtils.getTableKey(tables.keys, tableName) != null private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val key = getTableKey(map.keys, tableName) + val key = SqlActionUtils.getTableKey(map.keys, tableName) return map[key] } - private fun getTableKey(keys: Set, tableName: String) : String?{ - /* - * SQL is not case sensitivity, table/column must ignore case sensitivity. - * No, this is not really true... - * Usually, names are lowered-cased by the DB, unless quoted in "": - * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. - * - */ - val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } - if (!tableName.contains(".") && tableNameKey == null){ - //input name might be without schema, so check for partial match - val candidates = keys.filter { it.endsWith(".${tableName}", true) } - if(candidates.size > 1){ - throw IllegalArgumentException("Ambiguity." + - " More than one candidate of table called $tableName." + - " Values: ${candidates.joinToString(", ")}") - } - if(candidates.size == 1){ - return candidates[0] - } - } - return tableNameKey - } + fun getTable(tableName: String, useExtraConstraints: Boolean): Table { val data = if (useExtraConstraints) extendedTables else tables - val tableNameKey = getTableKey(data.keys, tableName) + val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) ?: throw IllegalArgumentException("No table called $tableName") return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt index a8dd1e5c35..93497032c7 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt @@ -7,6 +7,9 @@ import org.evomaster.dbconstraint.TableConstraint * Should be immutable */ data class Table( + /** + * This usually would be fully qualified, ie, including schema + */ val name: String, val columns: Set, diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index 9e0290b981..f4bca4a9b2 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -69,7 +69,7 @@ class ResourceNodeWithDbTest { assertEquals(6, cluster.getCluster().size) // table in db - assertTrue(cluster.getTableInfo().keys.containsAll(setOf("RFOO", "RBAR", "RXYZ"))) + assertTrue(cluster.getTableInfo().keys.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ"))) // data in db assertEquals(2, cluster.getDataInDb("RFOO")?.size) @@ -79,7 +79,7 @@ class ResourceNodeWithDbTest { val rfooNode = cluster.getResourceNode("/v3/api/rfoo") assertNotNull(rfooNode) rfooNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) + assertTrue(derivedMap.keys.contains("PUBLIC.RFOO"), "Keys: ${derivedMap.keys.joinToString(", ")}") assertEquals(1, paramToTable.size) assertTrue(paramToTable.values.first() is BodyParamRelatedToTable) (paramToTable.values.first() as BodyParamRelatedToTable).apply { From 5707132912407e0ac591bf8cd38a55d6c8bedb81 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 11:07:00 +0800 Subject: [PATCH 11/50] add calculateStringSimilarityScoreWithTableName --- .../problem/util/inference/SimpleDeriveR2T.kt | 29 ++++++++++++++++--- .../org/evomaster/core/sql/SqlActionUtils.kt | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 4946699781..9538d9917d 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -10,11 +10,14 @@ import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.rest.resource.dependency.* import org.evomaster.core.problem.util.ParamUtil +import org.evomaster.core.problem.util.SimilarityAlgorithm import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap import org.evomaster.core.problem.util.StringSimilarityComparator import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR +import kotlin.math.max /** * process inference related to resource @@ -44,8 +47,13 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(allTables.isNotEmpty()){ resourceNode.getAllSegments(flatten = true).forEach { seg -> ParamUtil.parseParams(seg).forEachIndexed stop@{ sindex, token -> - //check whether any table name matches token - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) }.asSequence().sortedBy { e->e.second } + /* + check whether any table name matches token + table name might contain schema, + */ + val matchedMap = allTables.keys.map { + Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) + }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { @@ -78,7 +86,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(reftypes.isNotEmpty()){ reftypes.forEach { type-> if(!resourceNode.isPartOfStaticTokens(type) && allTables.isNotEmpty()){ - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, type)) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, type)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ @@ -92,7 +100,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { //1.3 derive resource to tables based on tokens on POST action resourceNode.actions.filter {it.verb == HttpVerb.POST }.forEach { post-> post.tokens.values.filter { !resourceNode.getName().toLowerCase().contains(it.getKey().toLowerCase()) }.forEach { atoken-> - val matchedMap = allTables.keys.map { Pair(it, StringSimilarityComparator.stringSimilarityScore(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } matchedMap.last().apply { if(second >= StringSimilarityComparator.SimilarityThreshold){ resourceNode.resourceToTable.derivedMap.getOrPut(first){ @@ -107,6 +115,19 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { deriveParamsToTable(resourceNode.paramsInfo, resourceNode, allTables) } + /** + * compare [content] with given [tableName] + * tableName can contain info of schema, + * in this case we also calculate similarity with the last segment + * then return the higher similarity + */ + private fun calculateStringSimilarityScoreWithTableName(tableName: String, content: String): Double{ + val score = StringSimilarityComparator.stringSimilarityScore(tableName, content) + if (!tableName.contains(SCHEMA_TABLE_SEPARATOR)) return score + + val lastScore = StringSimilarityComparator.stringSimilarityScore(tableName.split(SCHEMA_TABLE_SEPARATOR).last(), content) + return max(score, lastScore) + } fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ mapParamInfo.forEach { (paramId, paramInfo) -> diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 10e2504fc7..202a5e7da3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -12,6 +12,8 @@ import org.evomaster.core.sql.schema.Table object SqlActionUtils { + const val SCHEMA_TABLE_SEPARATOR = "." + private val log: Logger = LoggerFactory.getLogger(SqlActionUtils::class.java) fun verifyForeignKeys(actions: List): Boolean { From 434bef478a02ffd1c37d47628c40f3073dd5b929 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 11:29:44 +0800 Subject: [PATCH 12/50] fix --- .../problem/util/inference/SimpleDeriveR2T.kt | 39 ++++++++++++------- .../rest/resource/ResourceNodeWithDbTest.kt | 14 ++++--- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 9538d9917d..9b91a085b8 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -10,10 +10,10 @@ import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.rest.resource.dependency.* import org.evomaster.core.problem.util.ParamUtil -import org.evomaster.core.problem.util.SimilarityAlgorithm import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap import org.evomaster.core.problem.util.StringSimilarityComparator +import org.evomaster.core.problem.util.StringSimilarityComparator.stringSimilarityScore import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR @@ -52,7 +52,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { table name might contain schema, */ val matchedMap = allTables.keys.map { - Pair(it, StringSimilarityComparator.stringSimilarityScore(it, token)) + Pair(it, calculateStringSimilarityScoreWithTableName(it, token)) }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ @@ -65,7 +65,9 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } val matchedPropertyMap = allTables.flatMap { t->t.value.columns.filter { c-> !ParamUtil.isGeneralName(c.name) }.map { c->Pair(t.value.name, c.name) } } - .map { p-> Pair(p.first, Pair(p.second,StringSimilarityComparator.stringSimilarityScore(p.second, token))) }.asSequence().sortedBy { e->e.second.second } + .map { p-> Pair(p.first, + Pair(p.second, calculateStringSimilarityScoreWithTableName(p.second, token) + )) }.asSequence().sortedBy { e->e.second.second } if(matchedPropertyMap.lastOrNull() != null && matchedPropertyMap.last().second.second >= StringSimilarityComparator.SimilarityThreshold){ matchedPropertyMap.filter { it.second.second == matchedPropertyMap.last().second.second }.forEach { @@ -122,11 +124,18 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * then return the higher similarity */ private fun calculateStringSimilarityScoreWithTableName(tableName: String, content: String): Double{ - val score = StringSimilarityComparator.stringSimilarityScore(tableName, content) - if (!tableName.contains(SCHEMA_TABLE_SEPARATOR)) return score - - val lastScore = StringSimilarityComparator.stringSimilarityScore(tableName.split(SCHEMA_TABLE_SEPARATOR).last(), content) - return max(score, lastScore) + var score = stringSimilarityScore(tableName, content) // full A vs full B + val shortContent = if (content.contains(SCHEMA_TABLE_SEPARATOR)) content.split(SCHEMA_TABLE_SEPARATOR).last() else null + val shortTable = if (tableName.contains(SCHEMA_TABLE_SEPARATOR)) tableName.split(SCHEMA_TABLE_SEPARATOR).last() else null + if (shortContent == null && shortTable == null) return score + + if (shortContent != null) + score = max(score, stringSimilarityScore(tableName, shortContent)) // full A vs short B + if (shortTable != null) + score = max(score, stringSimilarityScore(shortTable, content)) // short A vs full B + if (shortTable != null && shortContent != null) + score = max(score, stringSimilarityScore(shortTable, shortContent)) + return score } fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ @@ -211,16 +220,16 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { .map { Pair(it.name, if(ParamUtil.isGeneralName(it.name)) - t.foreignKeys.map { f-> StringSimilarityComparator.stringSimilarityScore(paramName, "${f.targetTable}${it.name}")} - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, "$tableName${it.name}")) + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTable}${it.name}")} + .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) + .plus(calculateStringSimilarityScoreWithTableName(paramName, "$tableName${it.name}")) .asSequence().sorted().last() else if(ParamUtil.isGeneralName(paramName)) - t.foreignKeys.map { f-> StringSimilarityComparator.stringSimilarityScore("${f.targetTable}$paramName", it.name)} - .plus(StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) - .plus(StringSimilarityComparator.stringSimilarityScore("$tableName$paramName", it.name)) + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTable}$paramName", it.name)} + .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) + .plus(calculateStringSimilarityScoreWithTableName("$tableName$paramName", it.name)) .asSequence().sorted().last() - else StringSimilarityComparator.stringSimilarityScore(paramName, it.name)) + else calculateStringSimilarityScoreWithTableName(paramName, it.name)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index f4bca4a9b2..cb1b084abd 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -14,6 +14,7 @@ import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.SqlActionUtils.isMatchingTableName import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -87,7 +88,8 @@ class ResourceNodeWithDbTest { fieldsMap.forEach { t, u -> assertEquals(1, u.derivedMap.size) u.derivedMap.forEach { ut, uu -> - assertEquals("RFOO", ut) +// assertEquals("RFOO", ut) + assertTrue(isMatchingTableName(ut, "RFOO")) assertEquals(uu.input.toLowerCase(), uu.targetMatched.toLowerCase()) } } @@ -97,16 +99,16 @@ class ResourceNodeWithDbTest { val rbarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}") assertNotNull(rbarNode) rbarNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) - assertTrue(derivedMap.keys.contains("RBAR")) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) } val rxyzNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}") assertNotNull(rxyzNode) rxyzNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("RFOO")) - assertTrue(derivedMap.keys.contains("RBAR")) - assertTrue(derivedMap.keys.contains("RXYZ")) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RXYZ") }) } } From 507defb08d8e2b5c9a29d09e07aa9c28ba1931db Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Thu, 2 Jan 2025 16:18:25 +0800 Subject: [PATCH 13/50] fix due to merge --- .../client/java/sql/internal/SqlExpressionEvaluator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java index 50744c842a..434ffb781a 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlExpressionEvaluator.java @@ -291,7 +291,7 @@ public void visit(ParenthesedSelect parenthesedSelect) { @Override public void visit(Column column) { String name = column.getColumnName(); - String table = sqlNameContext.getTableName(column); + String table = sqlNameContext.getFullyQualifiedTableName(column); Object value = dataRow.getValueByName(name, table); concreteValues.push(value); } From 04d66bd812d173cbccbcafe41ff41e969d8b77c3 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 10:08:11 +0100 Subject: [PATCH 14/50] refactoring table id in dto --- .../controller/internal/SutController.java | 4 +- .../db/sql/h2/H2SchemaExtractorTest.java | 56 ++++++------- .../internal/db/sql/h2/H2SqlHandlerTest.java | 4 +- .../sql/mysql/MySQLSchemaExtractorTest.java | 6 +- .../java/controller/api/dto/SqlDtoUtils.java | 28 +++---- .../api/dto/database/schema/TableDto.java | 14 +--- .../api/dto/database/schema/TableIdDto.java | 23 ++++++ .../client/java/sql/DbInfoExtractor.java | 37 ++++----- .../constraint/H2ConstraintExtractor.java | 12 +-- .../constraint/MySQLConstraintExtractor.java | 4 +- .../PostgresConstraintExtractor.java | 4 +- ...risticsCalculatorWithInsertionDtoTest.java | 2 +- .../internal/QueryResultTransformerTest.java | 2 +- .../core/solver/SMTConditionVisitor.kt | 2 +- .../core/solver/SMTLibZ3DbConstraintSolver.kt | 6 +- .../evomaster/core/solver/SmtLibGenerator.kt | 22 ++--- .../evomaster/core/sql/SqlInsertBuilder.kt | 8 +- .../core/sql/SqlInsertBuilderTest.kt | 2 +- .../sql/extract/h2/AlterTableCheckEnumTest.kt | 10 +-- .../sql/extract/h2/AlterTableCheckTest.kt | 10 +-- .../sql/extract/h2/AlterTableUniqueTest.kt | 12 +-- .../sql/extract/h2/CatwatchSqlExtractTest.kt | 18 ++--- .../extract/h2/CreateTableCheckEnumTest.kt | 10 +-- .../sql/extract/h2/CreateTableCheckTest.kt | 10 +-- .../h2/FeaturesServiceSqlExtractTest.kt | 14 ++-- .../core/sql/extract/h2/NewsSqlExtractTest.kt | 2 +- .../core/sql/extract/h2/OcvnExtractTest.kt | 4 +- .../extract/h2/ProxyPrintSqlExtractTest.kt | 80 +++++++++---------- .../sql/extract/h2/ScoutApiSqlExtractTest.kt | 56 ++++++------- .../mysql/AlterTableExtractCheckTest.kt | 10 +-- .../sql/extract/mysql/AlterTableUniqueTest.kt | 2 +- .../mysql/CreateTableBoundedNumberTest.kt | 4 +- .../extract/mysql/CreateTableCheckT1Test.kt | 8 +- .../sql/extract/mysql/CreateTableCheckTest.kt | 10 +-- .../sql/extract/mysql/CreateTableEnumTest.kt | 2 +- .../extract/mysql/CreateTableUniqueTest.kt | 2 +- .../postgres/AlterTableExtractCheckTest.kt | 10 +-- .../postgres/AlterTableExtractUniqueTest.kt | 12 +-- .../sql/extract/postgres/ArrayTypesTest.kt | 4 +- .../extract/postgres/CompositeTypesTest.kt | 4 +- .../postgres/CreateTableExtractCheckTest.kt | 10 +-- .../sql/extract/postgres/Ind0ExtractTest.kt | 26 +++--- .../postgres/ManySimilarToChecksTest.kt | 2 +- .../postgres/ObjectIdentifierTypesTest.kt | 4 +- .../core/sql/extract/postgres/PgLsnTest.kt | 4 +- .../sql/extract/postgres/SqlTextColumnTest.kt | 4 +- .../sql/multidb/ConflictingConstraintsTest.kt | 6 +- .../sql/multidb/ConflictingSchemasTest.kt | 6 +- .../core/sql/multidb/SecondSchemaTest.kt | 10 +-- 49 files changed, 307 insertions(+), 295 deletions(-) create mode 100644 client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 4924466a7e..02d621c34a 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -680,10 +680,10 @@ public final DbInfoDto getSqlDatabaseSchema() { if (fkMap.isEmpty()){ schemaDto.tables.forEach(t->{ - fkMap.putIfAbsent(t.name, new ArrayList<>()); + fkMap.putIfAbsent(t.id.name, new ArrayList<>()); if (t.foreignKeys!=null && !t.foreignKeys.isEmpty()){ t.foreignKeys.forEach(f->{ - fkMap.get(t.name).add(f.targetTable.toUpperCase()); + fkMap.get(t.id.name).add(f.targetTable.toUpperCase()); }); } }); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java index 38be462a73..831c325732 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SchemaExtractorTest.java @@ -32,7 +32,7 @@ public void testBasic() throws Exception { assertAll(() -> assertEquals("db_test", schema.name.toLowerCase()), () -> assertEquals(DatabaseType.H2, schema.databaseType), () -> assertEquals(1, schema.tables.size()), - () -> assertEquals("foo", schema.tables.get(0).name.toLowerCase()), + () -> assertEquals("foo", schema.tables.get(0).id.name.toLowerCase()), () -> assertEquals(1, schema.tables.get(0).columns.size()) ); } @@ -45,8 +45,8 @@ public void testTwoTables() throws Exception { assertNotNull(schema); assertEquals(2, schema.tables.size()); - assertTrue(schema.tables.stream().map(t -> t.name.toLowerCase()).anyMatch(n -> n.equals("foo"))); - assertTrue(schema.tables.stream().map(t -> t.name.toLowerCase()).anyMatch(n -> n.equals("bar"))); + assertTrue(schema.tables.stream().map(t -> t.id.name.toLowerCase()).anyMatch(n -> n.equals("foo"))); + assertTrue(schema.tables.stream().map(t -> t.id.name.toLowerCase()).anyMatch(n -> n.equals("bar"))); } @@ -135,8 +135,8 @@ public void testBasicForeignKey() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(2, schema.tables.size()); - TableDto bar = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Bar")).findAny().get(); - TableDto foo = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto bar = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Bar")).findAny().get(); + TableDto foo = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(0, bar.foreignKeys.size()); assertEquals(1, foo.foreignKeys.size()); @@ -184,7 +184,7 @@ public void testColumnUpperBoundConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -212,7 +212,7 @@ public void testTableConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -236,7 +236,7 @@ public void testPrimaryKey() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -257,7 +257,7 @@ public void testEnumStringConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(2, fooTable.columns.size()); @@ -282,7 +282,7 @@ public void testEnumBooleanConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -306,7 +306,7 @@ public void testEnumIntegerConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -331,7 +331,7 @@ public void testEnumTinyIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -355,7 +355,7 @@ public void testEnumSmallIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -380,7 +380,7 @@ public void testEnumBigIntConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -404,7 +404,7 @@ public void testEnumDoubleConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -429,7 +429,7 @@ public void testEnumRealConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -453,7 +453,7 @@ public void testEnumDecimalConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -475,7 +475,7 @@ public void testEnumCharConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -498,7 +498,7 @@ public void testEnumLikeConstraint() throws Exception { assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -520,7 +520,7 @@ public void testCreateEnumIntColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -546,7 +546,7 @@ public void testCreateEnumColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -571,7 +571,7 @@ public void testEnumColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - TableDto fooTable = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("Foo")).findAny().get(); + TableDto fooTable = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("Foo")).findAny().get(); assertEquals(1, fooTable.columns.size()); @@ -598,7 +598,7 @@ public void testArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -621,7 +621,7 @@ public void testMultidimensionalArrayOfTwoDimensions() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -644,7 +644,7 @@ public void testMultidimensionalArrayOfThreeDimensions() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -667,7 +667,7 @@ public void testBooleanNonlArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -690,7 +690,7 @@ public void testVarCharArray() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); @@ -714,7 +714,7 @@ public void testIntegerArrayWithMaxLengthColumn() throws Exception { DbInfoDto schema = DbInfoExtractor.extract(getConnection()); assertEquals(1, schema.tables.size()); - Optional fooTableOptional = schema.tables.stream().filter(t -> t.name.equalsIgnoreCase("foo")).findAny(); + Optional fooTableOptional = schema.tables.stream().filter(t -> t.id.name.equalsIgnoreCase("foo")).findAny(); assertTrue(fooTableOptional.isPresent()); TableDto fooTable = fooTableOptional.get(); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java index ff25f92386..8e3eb30d10 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/h2/H2SqlHandlerTest.java @@ -28,7 +28,7 @@ public void testHandleNoRows() throws Exception { DbInfoExtractor dbInfoExtractor = new DbInfoExtractor(); DbInfoDto schema = dbInfoExtractor.extract(connection); assertEquals(1, schema.tables.size()); - assertEquals("Person".toUpperCase(), schema.tables.get(0).name.toUpperCase()); + assertEquals("Person".toUpperCase(), schema.tables.get(0).id.name.toUpperCase()); SqlHandler sqlHandler = new SqlHandler(null); sqlHandler.setConnection(connection); @@ -60,7 +60,7 @@ public void testHandleOneOrMoreRows() throws Exception { DbInfoExtractor dbInfoExtractor = new DbInfoExtractor(); DbInfoDto schema = dbInfoExtractor.extract(connection); assertEquals(1, schema.tables.size()); - assertEquals("Person".toUpperCase(), schema.tables.get(0).name.toUpperCase()); + assertEquals("Person".toUpperCase(), schema.tables.get(0).id.name.toUpperCase()); SqlScriptRunner.execCommand(connection, "INSERT INTO Person (person_id, first_name, last_name, age, email)\n" + "VALUES (1, 'John', 'Doe', 30, 'john.doe@example.com');"); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java index 010f1941e8..a0f05e9253 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/mysql/MySQLSchemaExtractorTest.java @@ -86,7 +86,7 @@ public void testCaseSensitivityOfExtractor() throws Exception { TableDto table = schema.tables.get(0); assertEquals("intColumn",table.columns.get(0).name); - assertEquals("TableNot", table.name); + assertEquals("TableNot", table.id.name); } @@ -163,7 +163,7 @@ public void testTwoSchemas() throws Exception { DbInfoDto schemaTest0 = DbInfoExtractor.extract(testUser0Connection); assertEquals("test", schemaTest0.name); assertEquals(1, schemaTest0.tables.size()); - assertEquals("my_table", schemaTest0.tables.get(0).name); + assertEquals("my_table", schemaTest0.tables.get(0).id.name); DbInfoDto newSchema = DbInfoExtractor.extract(testUser1Connection); // assertEquals("new_schema", newSchema.name); @@ -171,7 +171,7 @@ public void testTwoSchemas() throws Exception { //assertEquals(1, newSchema.tables.size()); //we are now fetching data for all schemas (/catalogs in MySQL) assertEquals(2, newSchema.tables.size()); - assertEquals("my_table", newSchema.tables.get(0).name); + assertEquals("my_table", newSchema.tables.get(0).id.name); SqlScriptRunner.execCommand(testUser1Connection, "DROP SCHEMA IF EXISTS new_schema"); deleteUserInDatabase(url, anotherTestUserName); diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java index fad84fbf8c..922a6884da 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/SqlDtoUtils.java @@ -18,10 +18,10 @@ public class SqlDtoUtils { * Return a fully qualifying name for input table, which can be used as id. */ public static String getId(TableDto dto){ - if(dto.schema == null){ - return dto.name; + if(dto.id.schema == null){ + return dto.id.name; } - return dto.schema + "." + dto.name; + return dto.id.schema + "." + dto.id.name; } /** @@ -51,34 +51,34 @@ public static boolean matchByName(TableDto dto, String name){ throw new IllegalArgumentException("Invalid table name identifier. Too many '.': " + name); } if(tokens.length == 1){ - return dto.name.equalsIgnoreCase(tokens[0]); + return dto.id.name.equalsIgnoreCase(tokens[0]); } if(tokens.length == 2){ - boolean mn = dto.name.equalsIgnoreCase(tokens[1]); + boolean mn = dto.id.name.equalsIgnoreCase(tokens[1]); if(!mn){ return false; } - if(dto.catalog == null && dto.schema == null){ + if(dto.id.catalog == null && dto.id.schema == null){ //there is no default schema, but DTO is unspecified, then false return tokens[0].equalsIgnoreCase("public"); - } else if(dto.catalog != null && dto.schema != null){ + } else if(dto.id.catalog != null && dto.id.schema != null){ //both specified... so look at schema - return tokens[0].equalsIgnoreCase(dto.schema); + return tokens[0].equalsIgnoreCase(dto.id.schema); } else { //only one specified, take it - if(dto.schema != null){ - return tokens[0].equalsIgnoreCase(dto.schema); + if(dto.id.schema != null){ + return tokens[0].equalsIgnoreCase(dto.id.schema); } else{ //this can be the case for MySQL - return tokens[0].equalsIgnoreCase(dto.catalog); + return tokens[0].equalsIgnoreCase(dto.id.catalog); } } } if(tokens.length == 3){ - return tokens[0].equalsIgnoreCase(dto.catalog) - && tokens[1].equalsIgnoreCase(dto.schema) - && tokens[2].equalsIgnoreCase(dto.name); + return tokens[0].equalsIgnoreCase(dto.id.catalog) + && tokens[1].equalsIgnoreCase(dto.id.schema) + && tokens[2].equalsIgnoreCase(dto.id.name); } //shouldn't be reached diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java index 48ff605fcb..8451a21aad 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableDto.java @@ -4,19 +4,7 @@ public class TableDto { - /** - * The schema this table belongs to. - * Note that databases like MySQL make no distinction between catalog and schema. - */ - public String schema; - - - public String catalog; - - /** - * The name of the table - */ - public String name; + public TableIdDto id; /** * A list of descriptions for each column in the table diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java new file mode 100644 index 0000000000..98ff593f88 --- /dev/null +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/TableIdDto.java @@ -0,0 +1,23 @@ +package org.evomaster.client.java.controller.api.dto.database.schema; + +public class TableIdDto { + + /** + * The schema this table belongs to. + * Note that databases like MySQL make no distinction between catalog and schema. + */ + public String schema; + + + /** + * A physical database can have several independent catalogs. + * For Postgres, can only access the catalog specified in the connection, ie., one per connection. + * MySQL can access all catalogs even with a single connection (they work more like schemas...) + */ + public String catalog; + + /** + * The name of the table + */ + public String name; +} diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java index 80c959ebfd..b6d8208c66 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbInfoExtractor.java @@ -68,7 +68,7 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl .findFirst(); if (!targetTable.isPresent()) { - throw new IllegalArgumentException("Foreign key in table " + table.name + + throw new IllegalArgumentException("Foreign key in table " + table.id.name + " pointing to non-existent table " + fk.get().targetTable); } @@ -77,14 +77,14 @@ private static void checkForeignKeyToAutoIncrementMissing(DbInfoDto schema, Tabl .collect(Collectors.toList()); if (pks.isEmpty()) { - throw new IllegalArgumentException("No PK in table " + targetTable.get().name + " that has FKs pointing to it"); + throw new IllegalArgumentException("No PK in table " + targetTable.get().id.name + " that has FKs pointing to it"); } for (ColumnDto pk : pks) { if (pk.autoIncrement || pk.foreignKeyToAutoIncrement) { throw new IllegalArgumentException("Column " + pk.name + " in table " + pk.table + " is auto-increment, although FK pointing to it does not mark it " + - "as autoincrement in " + column.name + " in " + table.name + "as autoincrement in " + column.name + " in " + table.id.name ); } } @@ -101,7 +101,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl if (!fk.isPresent()) { throw new IllegalArgumentException("No foreign key constraint for marked column " + - column.name + " in table " + table.name); + column.name + " in table " + table.id.name); } //TODO proper handling of multi-column PKs/FKs @@ -111,7 +111,7 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl .findFirst(); if (!targetTable.isPresent()) { - throw new IllegalArgumentException("Foreign key in table " + table.name + + throw new IllegalArgumentException("Foreign key in table " + table.id.name + " pointing to non-existent table " + fk.get().targetTable); } @@ -123,15 +123,15 @@ private static void checkForeignKeyToAutoIncrementPresent(DbInfoDto schema, Tabl if (pks.size() != 1) { throw new IllegalArgumentException("There must be only 1 PK in table " + - targetTable.get().name + " pointed by the FK-to-autoincrement " + - column.name + " in " + table.name + ". However, there were: " + pks.size()); + targetTable.get().id.name + " pointed by the FK-to-autoincrement " + + column.name + " in " + table.id.name + ". However, there were: " + pks.size()); } ColumnDto pk = pks.get(0); if (!pk.autoIncrement && !pk.foreignKeyToAutoIncrement) { throw new IllegalArgumentException("Column " + pk.name + " in table " + pk.table + " is not auto-increment, although FK pointing to it does mark it" + - "as autoincrement in " + column.name + " in " + table.name + "as autoincrement in " + column.name + " in " + table.id.name ); } } @@ -498,7 +498,7 @@ public static void addUniqueConstraintToColumn(TableDto tableDto, String columnN .filter(c -> c.name.equals(columnName)).findAny().orElse(null); if (columnDto == null) { - throw new IllegalArgumentException("Missing column DTO for column:" + tableDto.name + "." + columnName); + throw new IllegalArgumentException("Missing column DTO for column:" + tableDto.id.name + "." + columnName); } columnDto.unique = true; @@ -575,9 +575,10 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, TableDto tableDto = new TableDto(); schemaDto.tables.add(tableDto); - tableDto.name = tables.getString("TABLE_NAME"); - tableDto.schema = tableSchema; - tableDto.catalog = tableCatalog; + tableDto.id = new TableIdDto(); + tableDto.id.name = tables.getString("TABLE_NAME"); + tableDto.id.schema = tableSchema; + tableDto.id.catalog = tableCatalog; if (tableIds.contains(SqlDtoUtils.getId(tableDto))) { /* @@ -590,7 +591,7 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, Set pks = new HashSet<>(); SortedMap primaryKeySequence = new TreeMap<>(); - ResultSet rsPK = md.getPrimaryKeys(tableDto.catalog, tableDto.schema, tableDto.name); + ResultSet rsPK = md.getPrimaryKeys(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name); while (rsPK.next()) { String pkColumnName = rsPK.getString("COLUMN_NAME"); @@ -603,21 +604,21 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, tableDto.primaryKeySequence.addAll(primaryKeySequence.values()); - ResultSet columns = md.getColumns(tableDto.catalog, tableDto.schema, tableDto.name, null); + ResultSet columns = md.getColumns(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name, null); Set columnNames = new HashSet<>(); while (columns.next()) { ColumnDto columnDto = new ColumnDto(); tableDto.columns.add(columnDto); - columnDto.table = tableDto.name; + columnDto.table = tableDto.id.name; columnDto.name = columns.getString("COLUMN_NAME"); if (columnNames.contains(columnDto.name)) { /* * Perhaps we should throw a more specific exception than IllegalArgumentException */ - throw new IllegalArgumentException("Cannot handle repeated column " + columnDto.name + " in table " + tableDto.name); + throw new IllegalArgumentException("Cannot handle repeated column " + columnDto.name + " in table " + tableDto.id.name); } else { columnNames.add(columnDto.name); } @@ -652,7 +653,7 @@ private static void handleTableEntry(Connection connection, DbInfoDto schemaDto, columns.close(); - ResultSet fks = md.getImportedKeys(tableDto.catalog, tableDto.schema, tableDto.name); + ResultSet fks = md.getImportedKeys(tableDto.id.catalog, tableDto.id.schema, tableDto.id.name); while (fks.next()) { //TODO need to see how to handle case of multi-columns @@ -727,7 +728,7 @@ private static void extractMySQLColumn(DbInfoDto schemaDto, * corresponding [DATA_TYPE] column value. */ String sqlQuery = String.format("SELECT DATA_TYPE, table_schema from INFORMATION_SCHEMA.COLUMNS where\n" + - " table_schema = '%s' and table_name = '%s' and column_name= '%s' ", tableDto.schema, tableDto.name, columnDto.name); + " table_schema = '%s' and table_name = '%s' and column_name= '%s' ", tableDto.id.schema, tableDto.id.name, columnDto.name); try (Statement statement = connection.createStatement()) { ResultSet rs = statement.executeQuery(sqlQuery); if (rs.next()) { diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java index 90b8d5fdb8..bb5049c879 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/H2ConstraintExtractor.java @@ -94,8 +94,8 @@ private List extractTableConstraintsVersionTwoOrHigher(Connec List tableCheckExpressions = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { final String query = String.format("Select CONSTRAINT_CATALOG,CONSTRAINT_SCHEMA,CONSTRAINT_NAME,CONSTRAINT_TYPE From INFORMATION_SCHEMA.TABLE_CONSTRAINTS\n" + " where TABLE_CONSTRAINTS.TABLE_SCHEMA='%s' \n" @@ -192,8 +192,8 @@ private List extractTableConstraintsVersionOneOrLower(Connect List tableCheckExpressions = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { final String query = String.format("Select CONSTRAINT_TYPE, CHECK_EXPRESSION, COLUMN_LIST From INFORMATION_SCHEMA.CONSTRAINTS\n" + " where CONSTRAINTS.TABLE_SCHEMA='%s' \n" @@ -255,8 +255,8 @@ private List extractColumnConstraintsVersion1OrLower(Connecti List columnConstraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToH2.createStatement()) { diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java index 6836ca44ef..abd3ed59df 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/MySQLConstraintExtractor.java @@ -42,8 +42,8 @@ public List extract(Connection connectionToMySQL, DbInfoDto s List constraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables){ - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToMySQL.createStatement()) { String query = String.format("SELECT *\n" + " FROM information_schema.table_constraints\n" + diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java index 69e6cbe6af..896020990a 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/constraint/PostgresConstraintExtractor.java @@ -72,8 +72,8 @@ public List extract(Connection connectionToPostgres, DbInfoDt List constraints = new ArrayList<>(); for (TableDto tableDto : schemaDto.tables) { - String tableSchema = tableDto.schema; - String tableName = tableDto.name; + String tableSchema = tableDto.id.schema; + String tableName = tableDto.id.name; try (Statement statement = connectionToPostgres.createStatement()) { diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java index 224bfda896..b069264b5c 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java @@ -20,7 +20,7 @@ public class HeuristicsCalculatorWithInsertionDtoTest { private DbInfoDto createSchemaDtoWithFooTableAndXColumn(String xDataType){ DbInfoDto schemaDto = new DbInfoDto(); TableDto tableDto = new TableDto(); - tableDto.name = "Foo"; + tableDto.id.name = "Foo"; ColumnDto dto = new ColumnDto(); dto.name = "x"; dto.type = xDataType; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index 7a21398d19..b1994d7b03 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -23,7 +23,7 @@ public class QueryResultTransformerTest { private TableDto createTableDate(List columnTypes, List columnNames, String tableName){ assertEquals(columnTypes.size(), columnNames.size()); TableDto tableDto = new TableDto(); - tableDto.name = tableName; + tableDto.id.name = tableName; for (int i = 0; i < columnTypes.size(); i++){ ColumnDto dto = new ColumnDto(); dto.name = columnNames.get(i); diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt index 62aecc50ae..fb67f66103 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTConditionVisitor.kt @@ -115,7 +115,7 @@ class SMTConditionVisitor( */ private fun isAColumn(operand: String): Boolean { return tables.any { - it.name.equals(defaultTableName, ignoreCase = true) && + it.id.name.equals(defaultTableName, ignoreCase = true) && it.columns.any { column -> column.name.equals(operand, ignoreCase = true) } } } diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt index c0ad5b1bf1..8c44924e46 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt @@ -140,7 +140,7 @@ class SMTLibZ3DbConstraintSolver( } private fun isBoolean(table: Table, columnName: String?): Boolean { - val col = schemaDto.tables.first { it.name == table.name }.columns.first { it.name == columnName } + val col = schemaDto.tables.first { it.id.name == table.name }.columns.first { it.name == columnName } return col.type == "BOOLEAN" } @@ -167,10 +167,10 @@ class SMTLibZ3DbConstraintSolver( * @return The Table object. */ private fun findTableByName(schema: DbInfoDto, tableName: String): Table { - val tableDto = schema.tables.find { it.name.equals(tableName, ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals(tableName, ignoreCase = true) } ?: throw RuntimeException("Table not found: $tableName") return Table( - tableDto.name, + tableDto.id.name, findColumns(tableDto), // Convert columns from DTO findForeignKeys(tableDto) // TODO: Implement this method ) diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt index 9ada18e561..040a08691f 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt @@ -54,7 +54,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I */ private fun appendTableDefinitions(smt: SMTLib) { for (table in schema.tables) { - val dataTypeName = "${StringUtils.capitalization(table.name)}Row" + val dataTypeName = "${StringUtils.capitalization(table.id.name)}Row" // Declare datatype for the table smt.addNode( @@ -64,7 +64,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Declare constants for each row for (i in 1..numberOfRows) { smt.addNode( - DeclareConstSMTNode("${table.name.lowercase(Locale.getDefault())}$i", dataTypeName) + DeclareConstSMTNode("${table.id.name.lowercase(Locale.getDefault())}$i", dataTypeName) ) } } @@ -89,7 +89,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which unique constraints are added. */ private fun appendUniqueConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.unique) { val nodes = assertForDistinctField(column.name, tableName) @@ -127,7 +127,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @return The corresponding SMT node. */ private fun parseCheckExpression(table: TableDto, condition: SqlCondition, index: Int): SMTNode { - val visitor = SMTConditionVisitor(table.name.lowercase(Locale.getDefault()), emptyMap(), schema.tables, index) + val visitor = SMTConditionVisitor(table.id.name.lowercase(Locale.getDefault()), emptyMap(), schema.tables, index) return condition.accept(visitor, null) as SMTNode } @@ -163,7 +163,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendBooleanConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.type.equals("BOOLEAN", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -190,7 +190,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendTimestampConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) for (column in table.columns) { if (column.type.equals("TIMESTAMP", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -228,7 +228,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which primary key constraints are added. */ private fun appendPrimaryKeyConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.name.lowercase(Locale.getDefault()) + val tableName = table.id.name.lowercase(Locale.getDefault()) val primaryKeys = table.columns.filter { it.primaryKey } for (primaryKey in primaryKeys) { @@ -270,11 +270,11 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which foreign key constraints are added. */ private fun appendForeignKeyConstraints(smt: SMTLib, table: TableDto) { - val sourceTableName = table.name.lowercase(Locale.getDefault()) + val sourceTableName = table.id.name.lowercase(Locale.getDefault()) for (foreignKey in table.foreignKeys) { val referencedTable = findReferencedTable(foreignKey) - val referencedTableName = referencedTable.name.lowercase(Locale.getDefault()) + val referencedTableName = referencedTable.id.name.lowercase(Locale.getDefault()) val referencedColumnSelector = findReferencedPKSelector(referencedTable, foreignKey) for (sourceColumn in foreignKey.sourceColumns) { @@ -343,7 +343,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @return The referenced table. */ private fun findReferencedTable(foreignKey: ForeignKeyDto): TableDto { - return schema.tables.firstOrNull { it.name.equals(foreignKey.targetTable, ignoreCase = true) } + return schema.tables.firstOrNull { it.id.name.equals(foreignKey.targetTable, ignoreCase = true) } ?: throw RuntimeException("Referenced table not found: ${foreignKey.targetTable}") } @@ -479,7 +479,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Only add GetValueSMTNode for the mentioned tables for (table in schema.tables) { - val tableNameLower = table.name.lowercase(Locale.getDefault()) + val tableNameLower = table.id.name.lowercase(Locale.getDefault()) if (tablesMentioned.contains(tableNameLower)) { for (i in 1..numberOfRows) { smt.addNode(GetValueSMTNode("$tableNameLower$i")) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index bdd522b44a..49f870a523 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -193,7 +193,7 @@ class SqlInsertBuilder( // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } - ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.name} does not have a column called $cname") + ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.id.name} does not have a column called $cname") sourceColumns.add(c) } @@ -212,8 +212,8 @@ class SqlInsertBuilder( for (column in tableDto.columns) { - if (!column.table.equals(tableDto.name, ignoreCase = true)) { - throw IllegalArgumentException("Column in different table: ${column.table}!=${tableDto.name}") + if (!column.table.equals(tableDto.id.name, ignoreCase = true)) { + throw IllegalArgumentException("Column in different table: ${column.table}!=${tableDto.id.name}") } val newColumn = createColumnFrom(column, tableConstraints, schemaDto) @@ -599,7 +599,7 @@ class SqlInsertBuilder( private fun parseTableConstraints(t: TableDto): List { val tableConstraints = mutableListOf() - val tableName = t.name + val tableName = t.id.name for (sqlCheckExpression in t.tableCheckExpressions) { val builder = TableConstraintBuilder() val constraintDatabaseType = getConstraintDatabaseType(this.databaseType) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt index ed40fa4094..28ab5568a3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt @@ -1283,7 +1283,7 @@ class SqlInsertBuilderTest { val sqlInsertBuilder = SqlInsertBuilder(schemaDto, null) return sqlInsertBuilder.createSqlInsertionAction( - schemaDto.tables[0].name, + schemaDto.tables[0].id.name, setOf("*"), mutableListOf(), false, diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt index 2808b70409..f983b0afd6 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckEnumTest.kt @@ -22,14 +22,14 @@ class AlterTableCheckEnumTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "X" }) + assertTrue(schema.tables.any { it.id.name == "X" }) - assertEquals(10, schema.tables.first { it.name == "X" }.columns.size) + assertEquals(10, schema.tables.first { it.id.name == "X" }.columns.size) - assertTrue(schema.tables.first { it.name == "X" }.columns.any { it.name == "STATUS" }); + assertTrue(schema.tables.first { it.id.name == "X" }.columns.any { it.name == "STATUS" }); - assertEquals(1, schema.tables.first { it.name == "X" }.tableCheckExpressions.size) - assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "X" }.tableCheckExpressions.size) + assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.id.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt index a345410aee..4e4894ab98 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableCheckTest.kt @@ -22,14 +22,14 @@ class AlterTableCheckTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "PEOPLE" }) + assertTrue(schema.tables.any { it.id.name == "PEOPLE" }) - assertEquals(2, schema.tables.first { it.name == "PEOPLE" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "PEOPLE" }.columns.size) - assertTrue(schema.tables.first { it.name == "PEOPLE" }.columns.any { it.name == "AGE" }); + assertTrue(schema.tables.first { it.id.name == "PEOPLE" }.columns.any { it.name == "AGE" }); - assertEquals(1, schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions.size) - assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions.size) + assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt index 37cdefdbbb..4c714ac54b 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/AlterTableUniqueTest.kt @@ -20,16 +20,16 @@ class AlterTableUniqueTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(2, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "COUNTRIES" }, "missing table COUNTRIES") }, - Executable { assertTrue(schema.tables.any { it.name == "PASSPORTS" }, "missing table PASSPORTS") } + Executable { assertTrue(schema.tables.any { it.id.name == "COUNTRIES" }, "missing table COUNTRIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "PASSPORTS" }, "missing table PASSPORTS") } ) - assertEquals(true, schema.tables.filter { it.name == "PASSPORTS" }.first().columns.filter { it.name == "COUNTRY_ID"}.first().unique) - assertEquals(true, schema.tables.filter { it.name == "PASSPORTS" }.first().columns.filter { it.name == "PASSPORT_NUMBER"}.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "PASSPORTS" }.first().columns.filter { it.name == "COUNTRY_ID"}.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "PASSPORTS" }.first().columns.filter { it.name == "PASSPORT_NUMBER"}.first().unique) - assertEquals(1, schema.tables.filter { it.name == "PASSPORTS" }.first().tableCheckExpressions.size) - assertEquals("(\"PASSPORT_NUMBER\" > CAST(0 AS BIGINT))", schema.tables.filter { it.name == "PASSPORTS" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.filter { it.id.name == "PASSPORTS" }.first().tableCheckExpressions.size) + assertEquals("(\"PASSPORT_NUMBER\" > CAST(0 AS BIGINT))", schema.tables.filter { it.id.name == "PASSPORTS" }.first().tableCheckExpressions[0].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt index c0ebd13d7c..f352d964eb 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt @@ -29,17 +29,17 @@ class CatwatchSqlExtractTest : ExtractTestBaseH2(){ assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(5, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "CONTRIBUTOR" }) }, - Executable { assertTrue(schema.tables.any { it.name == "LANGUAGE_LIST" }) }, - Executable { assertTrue(schema.tables.any { it.name == "MAINTAINERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PROJECT" }) }, - Executable { assertTrue(schema.tables.any { it.name == "STATISTICS" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "CONTRIBUTOR" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "LANGUAGE_LIST" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "MAINTAINERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PROJECT" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "STATISTICS" }) } ) - assertEquals(listOf("ID", "ORGANIZATION_ID", "SNAPSHOT_DATE"), schema.tables.filter { it.name == "CONTRIBUTOR" }.first().primaryKeySequence) - assertEquals(listOf("ID", "SNAPSHOT_DATE"), schema.tables.filter { it.name == "STATISTICS" }.first().primaryKeySequence) - assertEquals(listOf("ID"), schema.tables.filter { it.name == "PROJECT" }.first().primaryKeySequence) - assertEquals(listOf(), schema.tables.filter { it.name == "MAINTAINERS" }.first().primaryKeySequence) + assertEquals(listOf("ID", "ORGANIZATION_ID", "SNAPSHOT_DATE"), schema.tables.filter { it.id.name == "CONTRIBUTOR" }.first().primaryKeySequence) + assertEquals(listOf("ID", "SNAPSHOT_DATE"), schema.tables.filter { it.id.name == "STATISTICS" }.first().primaryKeySequence) + assertEquals(listOf("ID"), schema.tables.filter { it.id.name == "PROJECT" }.first().primaryKeySequence) + assertEquals(listOf(), schema.tables.filter { it.id.name == "MAINTAINERS" }.first().primaryKeySequence) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt index f29aa3b00c..12ab81c0b2 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckEnumTest.kt @@ -22,14 +22,14 @@ class CreateTableCheckEnumTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "X" }) + assertTrue(schema.tables.any { it.id.name == "X" }) - assertEquals(10, schema.tables.first { it.name == "X" }.columns.size) + assertEquals(10, schema.tables.first { it.id.name == "X" }.columns.size) - assertTrue(schema.tables.first { it.name == "X" }.columns.any { it.name == "STATUS" }); + assertTrue(schema.tables.first { it.id.name == "X" }.columns.any { it.name == "STATUS" }); - assertEquals(1, schema.tables.first { it.name == "X" }.tableCheckExpressions.size) - assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "X" }.tableCheckExpressions.size) + assertEquals("(\"STATUS\" IN('A', 'B'))", schema.tables.first { it.id.name == "X" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt index f9e6c9dc6d..ca58b0bd29 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CreateTableCheckTest.kt @@ -22,14 +22,14 @@ class CreateTableCheckTest : ExtractTestBaseH2() { assertEquals("db_test", schema.name.lowercase()) assertEquals(DatabaseType.H2, schema.databaseType) - assertTrue(schema.tables.any { it.name == "PEOPLE" }) + assertTrue(schema.tables.any { it.id.name == "PEOPLE" }) - assertEquals(2, schema.tables.first { it.name == "PEOPLE" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "PEOPLE" }.columns.size) - assertTrue(schema.tables.first { it.name == "PEOPLE" }.columns.any { it.name == "AGE" }); + assertTrue(schema.tables.first { it.id.name == "PEOPLE" }.columns.any { it.name == "AGE" }); - assertEquals(1, schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions.size) - assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions.size) + assertEquals("(\"AGE\" <= 100)", schema.tables.first { it.id.name == "PEOPLE" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt index 19a8b03cc6..0f94aa456c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/FeaturesServiceSqlExtractTest.kt @@ -22,15 +22,15 @@ class FeaturesServiceSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(6, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSTRAINT_EXCLUDES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSTRAINT_REQUIRES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "FEATURE" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT_CONFIGURATION" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "CONSTRAINT_EXCLUDES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "CONSTRAINT_REQUIRES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "FEATURE" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT_CONFIGURATION" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }) } ) - assertEquals(listOf("IN_CONFIGURATIONS_ID", "ACTIVED_FEATURES_ID"), schema.tables.filter { it.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }.first().primaryKeySequence) + assertEquals(listOf("IN_CONFIGURATIONS_ID", "ACTIVED_FEATURES_ID"), schema.tables.filter { it.id.name == "PRODUCT_CONFIGURATION_ACTIVED_FEATURES" }.first().primaryKeySequence) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt index d16bffc068..e895ce36e3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/NewsSqlExtractTest.kt @@ -21,7 +21,7 @@ class NewsSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(1, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "NEWS_ENTITY" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "NEWS_ENTITY" }) } ) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt index cc79b3252c..94eceab9c0 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/OcvnExtractTest.kt @@ -64,7 +64,7 @@ class OcvnExtractTest : ExtractTestBaseH2() { ) for (name in tableNames) { - assertTrue(schema.tables.any { it.name.equals(name, true) }, "Missing table $name") + assertTrue(schema.tables.any { it.id.name.equals(name, true) }, "Missing table $name") } } @@ -77,7 +77,7 @@ class OcvnExtractTest : ExtractTestBaseH2() { val builder = SqlInsertBuilder(schema) val tableName = "FILE_CONTENT" - assertTrue(schema.tables.any { it.name.equals(tableName, true) }) + assertTrue(schema.tables.any { it.id.name.equals(tableName, true) }) val columnName = "BYTES" val actions = builder.createSqlInsertionAction(tableName, setOf(columnName)) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt index 9b83d1c220..f5b39f5475 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ProxyPrintSqlExtractTest.kt @@ -30,18 +30,18 @@ class ProxyPrintSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(15, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "ADMIN" }) }, - Executable { assertTrue(schema.tables.any { it.name == "CONSUMERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "DOCUMENTS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "DOCUMENTS_SPECS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "EMPLOYEES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "MANAGERS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "NOTIFICATION" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRICETABLES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "PRINT_REQUESTS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "REVIEWS" }) }, - Executable { assertTrue(schema.tables.any { it.name == "ROLES" }) }, - Executable { assertTrue(schema.tables.any { it.name == "USERS" }) } + Executable { assertTrue(schema.tables.any { it.id.name == "ADMIN" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "CONSUMERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "DOCUMENTS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "DOCUMENTS_SPECS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "EMPLOYEES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "MANAGERS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "NOTIFICATION" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRICETABLES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "PRINT_REQUESTS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "REVIEWS" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "ROLES" }) }, + Executable { assertTrue(schema.tables.any { it.id.name == "USERS" }) } ) @@ -50,38 +50,38 @@ class ProxyPrintSqlExtractTest : ExtractTestBaseH2() { * table USERS has a unique column USERNAME: * alter table users add constraint UK_r43af9ap4edm43mmtq01oddj6 unique (username); */ - assertEquals(true, schema.tables.find { it.name == "USERS" }!!.columns.find { it.name == "USERNAME" }!!.unique) + assertEquals(true, schema.tables.find { it.id.name == "USERS" }!!.columns.find { it.name == "USERNAME" }!!.unique) /** * BIGSERIAL are autoincrement fields */ - assertEquals(true, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINTSHOPS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "PRINT_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(true, schema.tables.filter { it.name == "REVIEWS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - - assertEquals(false, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "PASSWORD" }.first().nullable) - assertEquals(false, schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "USERNAME" }.first().nullable) - - assertEquals(listOf("PRINTSHOP_ID", "ITEM"), schema.tables.filter { it.name == "PRICETABLES" }.first().primaryKeySequence); - - assertEquals(true, schema.tables.filter { it.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "ADMIN" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "EMPLOYEES" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); assertEquals(true, schema.tables.filter { it.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "MANAGERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); - assertEquals(true, schema.tables.filter { it.name == "PRICETABLES" }.first().columns.filter { it.name == "PRINTSHOP_ID" }.first().foreignKeyToAutoIncrement); - - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) - assertEquals(false, schema.tables.filter { it.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINTSHOPS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "PRINT_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertEquals(true, schema.tables.filter { it.id.name == "REVIEWS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + + assertEquals(false, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "PASSWORD" }.first().nullable) + assertEquals(false, schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "USERNAME" }.first().nullable) + + assertEquals(listOf("PRINTSHOP_ID", "ITEM"), schema.tables.filter { it.id.name == "PRICETABLES" }.first().primaryKeySequence); + + assertEquals(true, schema.tables.filter { it.id.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "ADMIN" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "EMPLOYEES" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); assertEquals(true, schema.tables.filter { it.id.name == "CONSUMERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "MANAGERS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement); + assertEquals(true, schema.tables.filter { it.id.name == "PRICETABLES" }.first().columns.filter { it.name == "PRINTSHOP_ID" }.first().foreignKeyToAutoIncrement); + + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "NOTIFICATION" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "DOCUMENTS_SPECS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "PRINTING_SCHEMAS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) + assertEquals(false, schema.tables.filter { it.id.name == "REGISTER_REQUESTS" }.first().columns.filter { it.name == "ID" }.first().foreignKeyToAutoIncrement) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt index d2c0f226df..b57e7a4655 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/ScoutApiSqlExtractTest.kt @@ -20,41 +20,41 @@ class ScoutApiSqlExtractTest : ExtractTestBaseH2() { assertAll(Executable { assertEquals("db_test", schema.name.lowercase()) }, Executable { assertEquals(DatabaseType.H2, schema.databaseType) }, Executable { assertEquals(14, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY" }, "missing table ACTIVITY") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_DERIVED" }, "missing table ACTIVITY_DERIVED") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES" }, "missing table ACTIVITY_PROPERTIES") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES_MEDIA_FILE" }, "missing table ACTIVITY_PROPERTIES_MEDIA_FILE") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_PROPERTIES_TAG" }, "missing table ACTIVITY_PROPERTIES_TAG") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_RATING" }, "missing table ACTIVITY_RATING") }, - Executable { assertTrue(schema.tables.any { it.name == "ACTIVITY_RELATION" }, "missing table ACTIVITY_RELATION") }, - Executable { assertTrue(schema.tables.any { it.name == "MEDIA_FILE" }, "missing table MEDIA_FILE") }, - Executable { assertTrue(schema.tables.any { it.name == "MEDIA_FILE_KEYWORDS" }, "missing table MEDIA_FILE_KEYWORDS") }, - Executable { assertTrue(schema.tables.any { it.name == "SYSTEM_MESSAGE" }, "missing table SYSTEM_MESSAGE") }, - Executable { assertTrue(schema.tables.any { it.name == "TAG" }, "missing table TAG") }, - Executable { assertTrue(schema.tables.any { it.name == "TAG_DERIVED" }, "missing table TAG_DERIVED") }, - Executable { assertTrue(schema.tables.any { it.name == "USER_IDENTITY" }, "missing table USER_IDENTITY") }, - Executable { assertTrue(schema.tables.any { it.name == "USERS" }, "missing table USERS") } + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY" }, "missing table ACTIVITY") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_DERIVED" }, "missing table ACTIVITY_DERIVED") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES" }, "missing table ACTIVITY_PROPERTIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES_MEDIA_FILE" }, "missing table ACTIVITY_PROPERTIES_MEDIA_FILE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_PROPERTIES_TAG" }, "missing table ACTIVITY_PROPERTIES_TAG") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_RATING" }, "missing table ACTIVITY_RATING") }, + Executable { assertTrue(schema.tables.any { it.id.name == "ACTIVITY_RELATION" }, "missing table ACTIVITY_RELATION") }, + Executable { assertTrue(schema.tables.any { it.id.name == "MEDIA_FILE" }, "missing table MEDIA_FILE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "MEDIA_FILE_KEYWORDS" }, "missing table MEDIA_FILE_KEYWORDS") }, + Executable { assertTrue(schema.tables.any { it.id.name == "SYSTEM_MESSAGE" }, "missing table SYSTEM_MESSAGE") }, + Executable { assertTrue(schema.tables.any { it.id.name == "TAG" }, "missing table TAG") }, + Executable { assertTrue(schema.tables.any { it.id.name == "TAG_DERIVED" }, "missing table TAG_DERIVED") }, + Executable { assertTrue(schema.tables.any { it.id.name == "USER_IDENTITY" }, "missing table USER_IDENTITY") }, + Executable { assertTrue(schema.tables.any { it.id.name == "USERS" }, "missing table USERS") } ) - assertTrue(schema.tables.filter { it.name == "ACTIVITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "TAG" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "USER_IDENTITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertTrue(schema.tables.filter { it.name == "MEDIA_FILE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "ACTIVITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "USERS" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "TAG" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "USER_IDENTITY" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) + assertTrue(schema.tables.filter { it.id.name == "MEDIA_FILE" }.first().columns.filter { it.name == "ID" }.first().autoIncrement) - assertEquals(5, schema.tables.filter { it.name == "USERS" }.first().columns.size) + assertEquals(5, schema.tables.filter { it.id.name == "USERS" }.first().columns.size) - assertEquals(true, schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "publishing_activity_id".uppercase() }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "MEDIA_FILE" }.first().columns.filter { it.name == "uri".uppercase() }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "keyColumn".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().columns.filter { it.name == "publishing_activity_id".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "MEDIA_FILE" }.first().columns.filter { it.name == "uri".uppercase() }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "SYSTEM_MESSAGE" }.first().columns.filter { it.name == "keyColumn".uppercase() }.first().unique) - assertEquals(2, schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions.size) - assertEquals("(\"AGE_MAX\" <= 100)", schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[0].sqlCheckExpression) - assertEquals("(\"AGE_MIN\" <= 100)", schema.tables.filter { it.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[1].sqlCheckExpression) + assertEquals(2, schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions.size) + assertEquals("(\"AGE_MAX\" <= 100)", schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals("(\"AGE_MIN\" <= 100)", schema.tables.filter { it.id.name == "ACTIVITY_PROPERTIES" }.first().tableCheckExpressions[1].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt index 33ebf504f9..56c5d34558 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableExtractCheckTest.kt @@ -18,12 +18,12 @@ class AlterTableExtractCheckTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("people", ignoreCase = true) }) - assertEquals(2, schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.size) + assertEquals(2, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.size) assertTrue(schema.tables.first { - it.name.equals( + it.id.name.equals( "people", ignoreCase = true ) @@ -31,12 +31,12 @@ class AlterTableExtractCheckTest : ExtractTestBaseMySQL() { assertEquals( 1, - schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size + schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size ) assertEquals( "(age <= 100)", schema.tables.first { - it.name.equals( + it.id.name.equals( "people", ignoreCase = true ) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt index 85c5d9a451..9ec7d3a9e7 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/AlterTableUniqueTest.kt @@ -19,7 +19,7 @@ class AlterTableUniqueTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - val tableDto = schema.tables.find { it.name.equals("suppliers", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("suppliers", ignoreCase = true) } assertNotNull(tableDto) assertEquals(4, tableDto!!.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt index d24033f14d..165b124d3b 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt @@ -25,9 +25,9 @@ class CreateTableBoundedNumberTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("BoundedNumberTable", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("BoundedNumberTable", ignoreCase = true) }) - val columns = schema.tables.first { it.name.equals("BoundedNumberTable", ignoreCase = true) }.columns + val columns = schema.tables.first { it.id.name.equals("BoundedNumberTable", ignoreCase = true) }.columns columns.apply { assertEquals(8, size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt index 1fa159a507..3ceb7abd1f 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckT1Test.kt @@ -18,12 +18,12 @@ class CreateTableCheckT1Test : ExtractTestBaseMySQL() { assertEquals("test", schema.name) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("t1", ignoreCase = true) }) - assertEquals(3, schema.tables.first { it.name.equals("t1", ignoreCase = true) }.columns.size) + assertTrue(schema.tables.any { it.id.name.equals("t1", ignoreCase = true) }) + assertEquals(3, schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.columns.size) - assertEquals(listOf("c1", "c2", "c3"),schema.tables.first { it.name.equals("t1", ignoreCase = true) }.columns.map { it.name }) + assertEquals(listOf("c1", "c2", "c3"),schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.columns.map { it.name }) - schema.tables.first { it.name.equals("t1", ignoreCase = true) }.tableCheckExpressions.apply { + schema.tables.first { it.id.name.equals("t1", ignoreCase = true) }.tableCheckExpressions.apply { assertEquals(6, size) /* https://dev.mysql.com/doc/refman/8.0/en/create-table-check-constraints.html diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt index c23ed8bad2..20d48c2d59 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableCheckTest.kt @@ -18,16 +18,16 @@ class CreateTableCheckTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people", ignoreCase = true) }) + assertTrue(schema.tables.any { it.id.name.equals("people", ignoreCase = true) }) - assertEquals(2, schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.size) + assertEquals(2, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.size) - assertTrue(schema.tables.first { it.name.equals("people", ignoreCase = true) }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size) + assertEquals(1, schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions.size) assertEquals( "(age <= 100)", - schema.tables.first { it.name.equals("people", ignoreCase = true) }.tableCheckExpressions[0].sqlCheckExpression + schema.tables.first { it.id.name.equals("people", ignoreCase = true) }.tableCheckExpressions[0].sqlCheckExpression ) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt index 8c2e02db0f..e4cf633c1f 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt @@ -18,7 +18,7 @@ class CreateTableEnumTest : ExtractTestBaseMySQL() { assertNotNull(schema) - val tableDto = schema.tables.find { it.name.equals("shirts", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("shirts", ignoreCase = true) } assertNotNull(tableDto) assertEquals(2, tableDto!!.columns.size) assertEquals(1, tableDto.tableCheckExpressions.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt index 080f7dd84f..417b4c8ac3 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableUniqueTest.kt @@ -19,7 +19,7 @@ class CreateTableUniqueTest : ExtractTestBaseMySQL() { assertEquals("test", schema.name.toLowerCase()) assertEquals(DatabaseType.MYSQL, schema.databaseType) - val tableDto = schema.tables.find { it.name.equals("suppliers", ignoreCase = true) } + val tableDto = schema.tables.find { it.id.name.equals("suppliers", ignoreCase = true) } assertNotNull(tableDto) assertEquals(4, tableDto!!.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt index d882b8d12b..a922e2ba33 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractCheckTest.kt @@ -22,14 +22,14 @@ class AlterTableExtractCheckTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "people" }) + assertTrue(schema.tables.any { it.id.name == "people" }) - assertEquals(2, schema.tables.first { it.name == "people" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "people" }.columns.size) - assertTrue(schema.tables.first { it.name == "people" }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name == "people" }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name == "people" }.tableCheckExpressions.size) - assertEquals("(age <= 100)", schema.tables.first { it.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "people" }.tableCheckExpressions.size) + assertEquals("(age <= 100)", schema.tables.first { it.id.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt index bafe886e2e..33a8ce684a 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/AlterTableExtractUniqueTest.kt @@ -20,16 +20,16 @@ class AlterTableExtractUniqueTest : ExtractTestBasePostgres() { assertAll(Executable { assertEquals("postgres", schema.name.toLowerCase()) }, Executable { assertEquals(DatabaseType.POSTGRES, schema.databaseType) }, Executable { assertEquals(2, schema.tables.size) }, - Executable { assertTrue(schema.tables.any { it.name == "countries" }, "missing table COUNTRIES") }, - Executable { assertTrue(schema.tables.any { it.name == "passports" }, "missing table PASSPORTS") } + Executable { assertTrue(schema.tables.any { it.id.name == "countries" }, "missing table COUNTRIES") }, + Executable { assertTrue(schema.tables.any { it.id.name == "passports" }, "missing table PASSPORTS") } ) - assertEquals(true, schema.tables.filter { it.name == "passports" }.first().columns.filter { it.name == "country_id" }.first().unique) - assertEquals(true, schema.tables.filter { it.name == "passports" }.first().columns.filter { it.name == "passport_number" }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "passports" }.first().columns.filter { it.name == "country_id" }.first().unique) + assertEquals(true, schema.tables.filter { it.id.name == "passports" }.first().columns.filter { it.name == "passport_number" }.first().unique) - assertEquals(1, schema.tables.filter { it.name == "passports" }.first().tableCheckExpressions.size) - assertEquals("(passport_number > 0)", schema.tables.filter { it.name == "passports" }.first().tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.filter { it.id.name == "passports" }.first().tableCheckExpressions.size) + assertEquals("(passport_number > 0)", schema.tables.filter { it.id.name == "passports" }.first().tableCheckExpressions[0].sqlCheckExpression) } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt index e86bc8142b..81d905d9e6 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ArrayTypesTest.kt @@ -39,8 +39,8 @@ class ArrayTypesTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("ArrayTypes".lowercase()) }) - val table = schema.tables.find { it.name.equals("ArrayTypes".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("ArrayTypes".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("ArrayTypes".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("nonArrayColumn".lowercase()) }) val nonArrayColumnDto = table.columns.find { it.name.equals("nonArrayColumn".lowercase()) }!! diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt index 3b746e72de..f59dd42312 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CompositeTypesTest.kt @@ -73,8 +73,8 @@ class CompositeTypesTest : ExtractTestBasePostgres() { assertTrue(itemColumn.columnTypeIsComposite) - assertTrue(schema.tables.any { it.name.equals("on_hand".lowercase()) }) - val onHandTable = schema.tables.find { it.name.equals("on_hand".lowercase()) }!! + assertTrue(schema.tables.any { it.id.name.equals("on_hand".lowercase()) }) + val onHandTable = schema.tables.find { it.id.name.equals("on_hand".lowercase()) }!! assertTrue(onHandTable.columns[0].name.equals("item".lowercase())) assertTrue(onHandTable.columns[1].name.equals("count".lowercase())) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt index 0a6c2c35ac..28ca153b5a 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/CreateTableExtractCheckTest.kt @@ -22,14 +22,14 @@ class CreateTableExtractCheckTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "people" }) + assertTrue(schema.tables.any { it.id.name == "people" }) - assertEquals(2, schema.tables.first { it.name == "people" }.columns.size) + assertEquals(2, schema.tables.first { it.id.name == "people" }.columns.size) - assertTrue(schema.tables.first { it.name == "people" }.columns.any { it.name == "age" }); + assertTrue(schema.tables.first { it.id.name == "people" }.columns.any { it.name == "age" }); - assertEquals(1, schema.tables.first { it.name == "people" }.tableCheckExpressions.size) - assertEquals("(age <= 100)", schema.tables.first { it.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) + assertEquals(1, schema.tables.first { it.id.name == "people" }.tableCheckExpressions.size) + assertEquals("(age <= 100)", schema.tables.first { it.id.name == "people" }.tableCheckExpressions[0].sqlCheckExpression) } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt index 31526c9da3..14f2771625 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/Ind0ExtractTest.kt @@ -28,26 +28,26 @@ class Ind0ExtractTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name == "x" }) - assertTrue(schema.tables.any { it.name == "y" }) + assertTrue(schema.tables.any { it.id.name == "x" }) + assertTrue(schema.tables.any { it.id.name == "y" }) - assertEquals(11, schema.tables.first { it.name == "x" }.columns.size) - assertEquals(7, schema.tables.first { it.name == "y" }.columns.size) + assertEquals(11, schema.tables.first { it.id.name == "x" }.columns.size) + assertEquals(7, schema.tables.first { it.id.name == "y" }.columns.size) - assertTrue(schema.tables.first { it.name == "x" }.columns.any { it.type.equals("xml", true) }) - assertTrue(schema.tables.first { it.name == "y" }.columns.any { it.type.equals("jsonb", true) }) + assertTrue(schema.tables.first { it.id.name == "x" }.columns.any { it.type.equals("xml", true) }) + assertTrue(schema.tables.first { it.id.name == "y" }.columns.any { it.type.equals("jsonb", true) }) - assertEquals(4, schema.tables.first { it.name == "x" }.tableCheckExpressions.size) - assertEquals(1, schema.tables.first { it.name == "y" }.tableCheckExpressions.size) + assertEquals(4, schema.tables.first { it.id.name == "x" }.tableCheckExpressions.size) + assertEquals(1, schema.tables.first { it.id.name == "y" }.tableCheckExpressions.size) // constraints on X table - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text]))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((status = 'B'::text) = (p_at IS NOT NULL))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((f_id ~~ 'hi'::text) OR (f_id ~~ '%foo%'::text) OR (f_id ~~ '%foo%x%'::text) OR (f_id ~~ '%bar%'::text) OR (f_id ~~ '%bar%y%'::text) OR (f_id ~~ '%hello%'::text))") }) - assertTrue(schema.tables.first { it.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(w_id ~ similar_to_escape('/foo/__/bar/(left|right)/[0-9]{4}-[0-9]{2}-[0-9]{2}(/[0-9]*)?'::text))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text]))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((status = 'B'::text) = (p_at IS NOT NULL))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("((f_id ~~ 'hi'::text) OR (f_id ~~ '%foo%'::text) OR (f_id ~~ '%foo%x%'::text) OR (f_id ~~ '%bar%'::text) OR (f_id ~~ '%bar%y%'::text) OR (f_id ~~ '%hello%'::text))") }) + assertTrue(schema.tables.first { it.id.name == "x" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(w_id ~ similar_to_escape('/foo/__/bar/(left|right)/[0-9]{4}-[0-9]{2}-[0-9]{2}(/[0-9]*)?'::text))") }) // constraints on Y table - assertTrue(schema.tables.first { it.name == "y" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text, 'C'::text, 'D'::text, 'E'::text]))") }) + assertTrue(schema.tables.first { it.id.name == "y" }.tableCheckExpressions.any { it.sqlCheckExpression.equals("(status = ANY (ARRAY['A'::text, 'B'::text, 'C'::text, 'D'::text, 'E'::text]))") }) //TODO check the 3 views diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt index 70c3641859..f91b9bb1d0 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt @@ -24,7 +24,7 @@ class ManySimilarToChecksTest : ExtractTestBasePostgres() { fun testManySimilarToPatternsSchemaExtraction() { val schema = DbInfoExtractor.extract(connection) - assertEquals(2, schema.tables.first { it.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) + assertEquals(2, schema.tables.first { it.id.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) val builder = SqlInsertBuilder(schema) val emailTable = builder.getTable("email_table", useExtraConstraints = true) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt index 8bf843a52d..c03f3ae7cb 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ObjectIdentifierTypesTest.kt @@ -44,8 +44,8 @@ class ObjectIdentifierTypesTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("ObjectIdentifierTypes".lowercase()) }) - val table = schema.tables.find { it.name.equals("ObjectIdentifierTypes".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("ObjectIdentifierTypes".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("ObjectIdentifierTypes".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("oidColumn".lowercase()) }) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt index 957eb06df6..0f576e4929 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/PgLsnTest.kt @@ -28,8 +28,8 @@ class PgLsnTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("PgLsnType".lowercase()) }) - val table = schema.tables.find { it.name.equals("PgLsnType".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("PgLsnType".lowercase()) }) + val table = schema.tables.find { it.id.name.equals("PgLsnType".lowercase()) } assertTrue(table!!.columns.any { it.name.equals("pglsnColumn".lowercase()) }) val nonArrayColumnDto = table.columns.find { it.name.equals("pglsnColumn".lowercase()) }!! diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt index 4aa04446a6..89f1d36a0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/SqlTextColumnTest.kt @@ -28,8 +28,8 @@ class SqlTextColumnTest : ExtractTestBasePostgres() { assertEquals("postgres", schema.name.lowercase()) assertEquals(DatabaseType.POSTGRES, schema.databaseType) - assertTrue(schema.tables.any { it.name.equals("people".lowercase()) }) - val peopleTable = schema.tables.find { it.name.equals("people".lowercase()) } + assertTrue(schema.tables.any { it.id.name.equals("people".lowercase()) }) + val peopleTable = schema.tables.find { it.id.name.equals("people".lowercase()) } val idColumn = peopleTable!!.columns[0] val nameColumn = peopleTable.columns[1] diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt index 1023919e4c..d18c5a1f15 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingConstraintsTest.kt @@ -12,10 +12,10 @@ class ConflictingConstraintsTest : MultiDbTestBase() { val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val first = info.tables.find { it.schema.equals("first",true) }!! - val second = info.tables.find { it.schema.equals("other",true) }!! + val first = info.tables.find { it.id.schema.equals("first",true) }!! + val second = info.tables.find { it.id.schema.equals("other",true) }!! - assertEquals(first.name, second.name) + assertEquals(first.id.name, second.id.name) assertEquals(2, first.columns.size) assertTrue(first.columns.any { it.name.equals("x", true) }) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt index b15e1b5af7..a82442e366 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/ConflictingSchemasTest.kt @@ -13,10 +13,10 @@ class ConflictingSchemasTest : MultiDbTestBase() { val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val first = info.tables.find { it.schema.equals("first",true) }!! - val second = info.tables.find { it.schema.equals("other",true) }!! + val first = info.tables.find { it.id.schema.equals("first",true) }!! + val second = info.tables.find { it.id.schema.equals("other",true) }!! - assertEquals(first.name, second.name) + assertEquals(first.id.name, second.id.name) assertEquals(2, first.columns.size) assertEquals(2, second.columns.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt index 985a70a9a4..047a0c0ca2 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/multidb/SecondSchemaTest.kt @@ -14,14 +14,14 @@ class SecondSchemaTest : MultiDbTestBase(){ val info = DbInfoExtractor.extract(connection) assertEquals(name.lowercase(), info.name.lowercase()) assertEquals(2, info.tables.size) - val foo = info.tables.find { it.name.lowercase() == "foo" }!! - val bar = info.tables.find { it.name.lowercase() == "bar" }!! + val foo = info.tables.find { it.id.name.lowercase() == "foo" }!! + val bar = info.tables.find { it.id.name.lowercase() == "bar" }!! if(databaseType == DatabaseType.MYSQL){ - assertEquals(name.lowercase(), foo.schema.lowercase()) + assertEquals(name.lowercase(), foo.id.schema.lowercase()) } else { - assertEquals("public", foo.schema.lowercase()) + assertEquals("public", foo.id.schema.lowercase()) } - assertEquals("other", bar.schema.lowercase()) + assertEquals("other", bar.id.schema.lowercase()) } override fun getSchemaLocation() = "/sql_schema/multidb/secondschema.sql" From ba60dfc23b8cb20e3a3e42a983ea80308015cd90 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 12:34:23 +0100 Subject: [PATCH 15/50] fix for some tests --- .../java/controller/internal/SutController.java | 6 +++++- .../dto/database/schema/ExtraConstraintsDto.java | 2 +- ...HeuristicsCalculatorWithInsertionDtoTest.java | 2 ++ .../sql/internal/QueryResultTransformerTest.java | 2 ++ .../org/evomaster/core/sql/SqlInsertBuilder.kt | 16 +++++++++++----- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 02d621c34a..de2e7e983f 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -24,6 +24,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.ExtraConstraintsDto; import org.evomaster.client.java.controller.api.dto.MockDatabaseDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.controller.api.dto.problem.RPCProblemDto; import org.evomaster.client.java.controller.api.dto.problem.rpc.*; import org.evomaster.client.java.sql.DbCleaner; @@ -1455,7 +1456,10 @@ protected UnitsInfoDto getUnitsInfoDto(UnitsInfoRecorder recorder){ ec.digitsInteger = c.getDigitsInteger(); ec.enumValuesAsStrings = c.getEnumValuesAsStrings() == null ? null : new ArrayList<>(c.getEnumValuesAsStrings()); ExtraConstraintsDto jpa = new ExtraConstraintsDto(); - jpa.tableName = c.getTableName(); + jpa.tableId = new TableIdDto(); + jpa.tableId.name = c.getTableName(); + jpa.tableId.schema = null; //TODO + jpa.tableId.catalog = null; //TODO jpa.columnName = c.getColumnName(); jpa.constraints = ec; return jpa; diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java index d6cf667615..ed4dcc7fab 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/schema/ExtraConstraintsDto.java @@ -8,7 +8,7 @@ */ public class ExtraConstraintsDto { - public String tableName; + public TableIdDto tableId; public String columnName; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java index b069264b5c..f2e30b8650 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/HeuristicsCalculatorWithInsertionDtoTest.java @@ -4,6 +4,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.DataRow; import org.evomaster.client.java.sql.QueryResult; import org.junit.jupiter.api.Test; @@ -20,6 +21,7 @@ public class HeuristicsCalculatorWithInsertionDtoTest { private DbInfoDto createSchemaDtoWithFooTableAndXColumn(String xDataType){ DbInfoDto schemaDto = new DbInfoDto(); TableDto tableDto = new TableDto(); + tableDto.id = new TableIdDto(); tableDto.id.name = "Foo"; ColumnDto dto = new ColumnDto(); dto.name = "x"; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index b1994d7b03..b1df973402 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -4,6 +4,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.QueryResult; import org.junit.jupiter.api.Test; @@ -23,6 +24,7 @@ public class QueryResultTransformerTest { private TableDto createTableDate(List columnTypes, List columnNames, String tableName){ assertEquals(columnTypes.size(), columnNames.size()); TableDto tableDto = new TableDto(); + tableDto.id = new TableIdDto(); tableDto.id.name = tableName; for (int i = 0; i < columnTypes.size(); i++){ ColumnDto dto = new ColumnDto(); diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 49f870a523..6a7517b16f 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -130,11 +130,11 @@ class SqlInsertBuilder( private fun validateExtraConstraintsHandle(schemaDto: DbInfoDto) { schemaDto.extraConstraintDtos.forEach { extraConstraintsDto -> - val table = tables.values.find { matchJpaName(it.name, extraConstraintsDto.tableName) } + val table = tables.values.find { matchJpaName(it.name, extraConstraintsDto.tableId.name) } if (table == null) { LoggingUtil.uniqueWarn( log, "Handling of extra constraints failed." + - " There is no SQL table called ${extraConstraintsDto.tableName}" + " There is no SQL table called ${extraConstraintsDto.tableId.name}" ) assert(false) } else { @@ -157,7 +157,7 @@ class SqlInsertBuilder( table: Table ): Table { val extrasForTable = schemaDto.extraConstraintDtos - .filter { matchJpaName(it.tableName, table.name) } + .filter { matchJpaName(it.tableId.name, table.name) } val columns = table.columns .map { column -> @@ -304,10 +304,16 @@ class SqlInsertBuilder( so, if match fails, we try again without _ */ private fun matchJpaName(original: String, jpaDefaultMadness: String): Boolean { - if (original.equals(jpaDefaultMadness, true)) { + /* + FIXME this will need to be refactored once we introduce TableId + */ + val name = if(original.contains(".")) original.substring(original.lastIndexOf(".") + 1, original.length) + else original + + if (name.equals(jpaDefaultMadness, true)) { return true } - return original.replace("_", "").equals(jpaDefaultMadness.replace("_", ""), true) + return name.replace("_", "").equals(jpaDefaultMadness.replace("_", ""), true) } private fun mergeConstraints(column: Column, extra: ExtraConstraintsDto): Column { From cd98db970a9cb0d9e33aa7aa7d6c026f4f4c86d4 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 12:56:14 +0100 Subject: [PATCH 16/50] introducing TableId --- .../core/solver/SMTLibZ3DbConstraintSolver.kt | 7 +-- .../org/evomaster/core/sql/SqlAction.kt | 48 ++-------------- .../core/sql/SqlActionTransformer.kt | 2 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 5 +- .../org/evomaster/core/sql/schema/Table.kt | 12 ++-- .../org/evomaster/core/sql/schema/TableId.kt | 57 +++++++++++++++++++ .../kotlin/org/evomaster/core/TestUtils.kt | 3 +- 7 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt index 8c44924e46..040cd44a22 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt @@ -14,10 +14,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.sql.SqlAction -import org.evomaster.core.sql.schema.Column -import org.evomaster.core.sql.schema.ColumnDataType -import org.evomaster.core.sql.schema.ForeignKey -import org.evomaster.core.sql.schema.Table +import org.evomaster.core.sql.schema.* import org.evomaster.solver.Z3DockerExecutor import org.evomaster.solver.smtlib.SMTLib import org.evomaster.solver.smtlib.value.* @@ -170,7 +167,7 @@ class SMTLibZ3DbConstraintSolver( val tableDto = schema.tables.find { it.id.name.equals(tableName, ignoreCase = true) } ?: throw RuntimeException("Table not found: $tableName") return Table( - tableDto.id.name, + TableId(tableDto.id.name) , //TODO other info, eg schema findColumns(tableDto), // Convert columns from DTO findForeignKeys(tableDto) // TODO: Implement this method ) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt index 0dffdc43dc..c2e671f9bc 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlAction.kt @@ -45,39 +45,6 @@ class SqlAction( */ val representExistingData: Boolean = false, - /** - * An id representing the connection to the database process. - * This could be the connecting URL. - * This is needed when dealing with multiple connection to different database processes, - * possibly running on different machines. - * If left null, default connection will be used (ie, assume there is only one). - */ - val connectionId: String? = null, - - /** - * At a high-level this represents a "catalog". - * Ie, a "logical" database instance. - * Note that a database process could have several logical databases. - * We do not use the term "catalog" though as Postgres and MySQL behave differently. - * In MySQL, can access different catalogs on same connection. - * This is not possible on Postgres. - * However, this latter has "schemas", which MySQL does not. - * In other words: - * - MySQL: this will always be empty - * - Postgres: this would be the catalog name. - * However, as in Postgres we cannot change catalog within same connection, the important info is in the - * [connectionId] entry. - * Even if we were to access 2 different catalogs in the same database process, we would need 2 different connections. - * As such, this parameter is technically redundant. - */ - val sealedGroupName: String? = null, - - /** - * In Postgres this represents a "schema", whereas it is a "catalog" for MySQL. - * In other words, this is used to group tables that can be indexed by this group name for disambiguation. - */ - val openGroupName: String? = null, - ) : EnvironmentAction(listOf()) { init { @@ -98,12 +65,6 @@ class SqlAction( } } - fun getFullQualifyingTableName() : String{ - if(openGroupName.isNullOrBlank()){ - return table.name - } - return "$openGroupName.${table.name}" - } private val genes: List = (computedGenes ?: selectedColumns.map { SqlActionGeneBuilder().buildGene(id, table, it) } @@ -136,9 +97,10 @@ class SqlAction( } override fun getName(): String { - val c = if(connectionId.isNullOrEmpty()) "" else "_$connectionId" - val s = if(openGroupName.isNullOrEmpty()) "" else "_$openGroupName" - val t = "_${table.name}" + val id = table.id + val c = if(id.connectionId.isNullOrEmpty()) "" else "_${id.connectionId}" + val s = if(id.openGroupName.isNullOrEmpty()) "" else "_${id.openGroupName}" + val t = "_${id.name}" return "SQL_Insert$c$s${t}_${selectedColumns.map { it.name }.sorted().joinToString("_")}" } @@ -155,7 +117,7 @@ class SqlAction( } override fun copyContent(): Action { - return SqlAction(table, selectedColumns, id, genes.map(Gene::copy), representExistingData, connectionId, sealedGroupName, openGroupName) + return SqlAction(table, selectedColumns, id, genes.map(Gene::copy), representExistingData) } fun geInsertionId(): Long { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt index 29277d4448..5f459d3efd 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionTransformer.kt @@ -35,7 +35,7 @@ object SqlActionTransformer { } - val insertion = InsertionDto().apply { targetTable = action.getFullQualifyingTableName() } + val insertion = InsertionDto().apply { targetTable = action.table.id.getFullQualifyingTableName() } for (g in action.seeTopGenes()) { if (g is SqlPrimaryKeyGene) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 6a7517b16f..9459deb8c3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -33,7 +33,7 @@ class SqlInsertBuilder( */ /** - * Information about tables, indexed by name + * Information about tables, indexed by id */ private val tables = mutableMapOf() @@ -112,7 +112,8 @@ class SqlInsertBuilder( for (tableDto in schemaDto.tables) { val id = SqlDtoUtils.getId(tableDto) val table = Table( - id, + //FIXME MySQL vs Postgres + TableId(tableDto.id.name, /*TODO*/ null, tableDto.id.catalog, tableDto.id.schema), tableToColumns[id]!!, tableToForeignKeys[id]!!, tableToConstraints[id]!! diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt index 93497032c7..f0192457cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt @@ -7,10 +7,8 @@ import org.evomaster.dbconstraint.TableConstraint * Should be immutable */ data class Table( - /** - * This usually would be fully qualified, ie, including schema - */ - val name: String, + + val id: TableId, val columns: Set, @@ -22,6 +20,12 @@ data class Table( */ val tableConstraints: Set = setOf() ){ + constructor(name: String, columns: Set, foreignKeys: Set, tableConstraints: Set = setOf()) + : this(TableId(name), columns, foreignKeys, tableConstraints) + + @Deprecated("Use id directly instead") + val name: String + get() = id.name fun primaryKeys() = columns.filter { it.primaryKey } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt new file mode 100644 index 0000000000..23f8169de3 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt @@ -0,0 +1,57 @@ +package org.evomaster.core.sql.schema + +data class TableId( + + /** + * The name of table + */ + val name: String, + + /** + * An id representing the connection to the database process. + * This could be the connecting URL. + * This is needed when dealing with multiple connection to different database processes, + * possibly running on different machines. + * If left null, default connection will be used (ie, assume there is only one). + */ + val connectionId: String? = null, + + /** + * At a high-level this represents a "catalog". + * Ie, a "logical" database instance. + * Note that a database process could have several logical databases. + * We do not use the term "catalog" though as Postgres and MySQL behave differently. + * In MySQL, can access different catalogs on same connection. + * This is not possible on Postgres. + * However, this latter has "schemas", which MySQL does not. + * In other words: + * - MySQL: this will always be empty + * - Postgres: this would be the catalog name. + * However, as in Postgres we cannot change catalog within same connection, the important info is in the + * [connectionId] entry. + * Even if we were to access 2 different catalogs in the same database process, we would need 2 different connections. + * As such, this parameter is technically redundant. + */ + val sealedGroupName: String? = null, + + /** + * In Postgres this represents a "schema", whereas it is a "catalog" for MySQL. + * In other words, this is used to group tables that can be indexed by this group name for disambiguation. + */ + val openGroupName: String? = null, +){ + + init { + if(name.contains(".")){ + throw IllegalArgumentException("Name contains a '.'. " + + "You sure you passed a simple name and not a fully qualified one? Value: $name") + } + } + + fun getFullQualifyingTableName() : String{ + if(openGroupName.isNullOrBlank()){ + return name + } + return "$openGroupName.${name}" + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt index b03910bee1..a46946016d 100644 --- a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt +++ b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt @@ -18,6 +18,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene +import org.evomaster.core.sql.schema.TableId object TestUtils { @@ -63,7 +64,7 @@ object TestUtils { // for rest problem fun generateFakeDbAction(pkId : Long, pkGeneUniqueId: Long, tableName : String = "Foo", intValue : Int =0, fkColumn : Column?=null, fkGene: SqlForeignKeyGene? = null) : SqlAction { val fooId = Column("Id", ColumnDataType.INTEGER, 10, primaryKey = true, databaseType = DatabaseType.H2) - val foo = Table(tableName, setOf(fooId), setOf()) + val foo = Table(TableId(tableName), setOf(fooId), setOf()) val integerGene = IntegerGene(fooId.name, intValue) val pkFoo = SqlPrimaryKeyGene(fooId.name, "Foo", integerGene, pkGeneUniqueId) if(fkColumn != null && fkGene != null) From cdb2a64d913c904720d87a2b3ed0a7e6b9885445 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Jan 2025 14:38:59 +0100 Subject: [PATCH 17/50] starting refactoring for TableId --- .../core/output/service/TestSuiteWriter.kt | 9 +++- .../enterprise/service/EnterpriseSampler.kt | 5 +- .../problem/util/inference/SimpleDeriveR2T.kt | 4 +- .../core/search/gene/sql/SqlForeignKeyGene.kt | 5 +- .../core/search/gene/sql/SqlPrimaryKeyGene.kt | 3 +- .../evomaster/core/search/service/Sampler.kt | 6 --- .../core/sql/SqlActionGeneBuilder.kt | 2 +- .../org/evomaster/core/sql/SqlActionUtils.kt | 4 +- .../evomaster/core/sql/SqlInsertBuilder.kt | 50 +++++++++---------- .../evomaster/core/sql/schema/ForeignKey.kt | 2 +- .../org/evomaster/core/sql/schema/TableId.kt | 26 ++++++++++ .../individual/RestIndividualStructureTest.kt | 2 +- .../evomaster/core/sql/DbActionUtilsTest.kt | 21 ++++---- .../DbDirectIntWithSqlEMTest.java | 3 +- .../e2etests/spring/rest/basic/BasicEMTest.kt | 3 +- .../DbDirectIntWithSqlEMTest.java | 3 +- 16 files changed, 87 insertions(+), 61 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 4753b28ae1..8ebd96c9b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -12,6 +12,7 @@ import org.evomaster.core.output.TestWriterUtils.handleDefaultStubForAsJavaOrKot import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy import org.evomaster.core.output.naming.TestCaseNamingStrategyFactory import org.evomaster.core.problem.api.ApiWsIndividual +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler @@ -21,6 +22,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.sql.schema.TableId import org.evomaster.test.utils.EMTestUtils import org.evomaster.test.utils.SeleniumEMUtils import org.evomaster.test.utils.js.JsLoader @@ -191,10 +193,13 @@ class TestSuiteWriter { } private fun handleResetDatabaseInput(solution: Solution<*>): String { + if(sampler !is EnterpriseSampler<*>){ + throw IllegalArgumentException("Not dealing with an enterprise application") + } if (!config.outputFormat.isJavaOrKotlin()) throw IllegalStateException("DO NOT SUPPORT resetDatabased for " + config.outputFormat) - val accessedTable = mutableSetOf() + val accessedTable = mutableSetOf() solution.individuals.forEach { e -> //TODO will need to be refactored when supporting Web Frontend if (e.individual is ApiWsIndividual) { @@ -206,7 +211,7 @@ class TestSuiteWriter { accessedTable.addAll(de.deletedData) } } - val all = sampler.extractFkTables(accessedTable) + val all = (sampler as EnterpriseSampler).extractFkTables(accessedTable) val tableNamesInSchema = remoteController.getCachedSutInfo() ?.sqlSchemaDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 04e3dd581a..b9d81618b0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.SutProblemException import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Individual import org.evomaster.core.search.service.Sampler +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -53,7 +54,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } } - fun sampleSqlInsertion(tableName: String, columns: Set): List { + fun sampleSqlInsertion(tableName: TableId, columns: Set): List { val extraConstraints = randomness.nextBoolean(apc.getExtraSqlDbConstraintsProbability()) val enableSingleInsertionForTable = randomness.nextBoolean(config.probOfEnablingSingleInsertionForTable) @@ -96,7 +97,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } } - override fun extractFkTables(tables: Set): Set { + fun extractFkTables(tables: Set): Set { if(sqlInsertBuilder == null || tables.isEmpty()) return tables return sqlInsertBuilder!!.extractFkTable(tables) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 9b91a085b8..f1649a039e 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -220,12 +220,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { .map { Pair(it.name, if(ParamUtil.isGeneralName(it.name)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTable}${it.name}")} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTableId}${it.name}")} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) .plus(calculateStringSimilarityScoreWithTableName(paramName, "$tableName${it.name}")) .asSequence().sorted().last() else if(ParamUtil.isGeneralName(paramName)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTable}$paramName", it.name)} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTableId}$paramName", it.name)} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) .plus(calculateStringSimilarityScoreWithTableName("$tableName$paramName", it.name)) .asSequence().sorted().last() diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt index 42883e4ad9..a884f2143e 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt @@ -9,6 +9,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.sql.schema.TableId /** * A gene specifically designed to handle Foreign Keys in SQL databases. @@ -25,9 +26,9 @@ class SqlForeignKeyGene( sourceColumn: String, val uniqueId: Long, /** - * The name of the table this FK points to + * The id of the table this FK points to */ - val targetTable: String, + val targetTable: TableId, val nullable: Boolean, /** * A negative value means this FK is not bound yet. diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt index c4d807bc68..c766d173f1 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt @@ -10,6 +10,7 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -19,7 +20,7 @@ import org.slf4j.LoggerFactory * This is important to check Foreign Keys referencing it. */ class SqlPrimaryKeyGene(name: String, - val tableName: String, + val tableName: TableId, val gene: Gene, /** * Important for the Foreign Keys referencing it. diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt index e19726d391..21b46d654d 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt @@ -195,12 +195,6 @@ abstract class Sampler : TrackOperator where T : Individual { } } - /** - * extract tables with additional FK tables - */ - open fun extractFkTables(tables: Set): Set{ - throw IllegalStateException("FK tables have not been not handled yet") - } /** * Return a list of pre-written individuals that will be added in the final solution. diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt index 4fe6afdd83..d65c7eb55e 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt @@ -54,7 +54,7 @@ class SqlActionGeneBuilder { column.autoIncrement -> SqlAutoIncrementGene(column.name) fk != null -> - SqlForeignKeyGene(column.name, id, fk.targetTable, column.nullable) + SqlForeignKeyGene(column.name, id, fk.targetTableId, column.nullable) else -> when (column.type) { // Man: TODO need to check diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 202a5e7da3..769e693bd4 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -432,8 +432,8 @@ object SqlActionUtils { val sorted = table.sortedWith( Comparator { o1, o2 -> when { - o1.foreignKeys.any { t-> t.targetTable.equals(o2.name,ignoreCase = true) } -> -1 - o2.foreignKeys.any { t-> t.targetTable.equals(o1.name,ignoreCase = true) } -> 1 + o1.foreignKeys.any { t-> t.targetTableId == o2.id } -> -1 + o2.foreignKeys.any { t-> t.targetTableId == o1.id } -> 1 else -> 0 } } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 9459deb8c3..6653f99dea 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -35,12 +35,12 @@ class SqlInsertBuilder( /** * Information about tables, indexed by id */ - private val tables = mutableMapOf() + private val tables = mutableMapOf() /** * Same as table, but possibly with extended info on constraints, derived from analyzing the SUT */ - private val extendedTables = mutableMapOf() + private val extendedTables = mutableMapOf() private val compositeTypes = mutableMapOf() @@ -87,33 +87,33 @@ class SqlInsertBuilder( compositeTypes[compositeTypeDto.name] = CompositeType(compositeTypeDto.name, columns) } - val tableToColumns = mutableMapOf>() - val tableToForeignKeys = mutableMapOf>() - val tableToConstraints = mutableMapOf>() + val tableToColumns = mutableMapOf>() + val tableToForeignKeys = mutableMapOf>() + val tableToConstraints = mutableMapOf>() // Then load all constraints and columns for (tableDto in schemaDto.tables) { val tableConstraints = parseTableConstraints(tableDto).toMutableList() val columns = generateColumnsFrom(tableDto, tableConstraints, schemaDto) - - tableToConstraints[SqlDtoUtils.getId(tableDto)] = tableConstraints.toSet() - tableToColumns[SqlDtoUtils.getId(tableDto)] = columns + val id = TableId.fromDto(databaseType, tableDto.id) + tableToConstraints[id] = tableConstraints.toSet() + tableToColumns[id] = columns } // After all columns are loaded, we can load foreign keys for (tableDto in schemaDto.tables) { - tableToForeignKeys[SqlDtoUtils.getId(tableDto)] = calculateForeignKeysFrom(tableDto, tableToColumns) + val id = TableId.fromDto(databaseType, tableDto.id) + tableToForeignKeys[id] = calculateForeignKeysFrom(tableDto, tableToColumns) } // Now we can create the tables for (tableDto in schemaDto.tables) { - val id = SqlDtoUtils.getId(tableDto) + val id = TableId.fromDto(schemaDto.databaseType, tableDto.id) val table = Table( - //FIXME MySQL vs Postgres - TableId(tableDto.id.name, /*TODO*/ null, tableDto.id.catalog, tableDto.id.schema), + id, tableToColumns[id]!!, tableToForeignKeys[id]!!, tableToConstraints[id]!! @@ -175,13 +175,14 @@ class SqlInsertBuilder( private fun calculateForeignKeysFrom( tableDto: TableDto, - tableToColumns: MutableMap> + tableToColumns: MutableMap> ): MutableSet { val fks = mutableSetOf() for (fk in tableDto.foreignKeys) { val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) + FIXME if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -641,14 +642,14 @@ class SqlInsertBuilder( - fun getTable(tableName: String, useExtraConstraints: Boolean): Table { + fun getTable(tableName: TableId, useExtraConstraints: Boolean): Table { val data = if (useExtraConstraints) extendedTables else tables - val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) - ?: throw IllegalArgumentException("No table called $tableName") +// val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) +// ?: throw IllegalArgumentException("No table called $tableName") - return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName") + return data[tableName] ?: throw IllegalArgumentException("No table called $tableName") } /** @@ -670,7 +671,7 @@ class SqlInsertBuilder( * test cases manually for EM */ fun createSqlInsertionAction( - tableName: String, + tableName: TableId, /** * Which columns to create data for. Default is all, ie *. * Notice that more columns might be added, eg, to satisfy non-null @@ -680,7 +681,7 @@ class SqlInsertBuilder( /** * used to avoid infinite recursion */ - history: MutableList = mutableListOf(), + history: MutableList = mutableListOf(), /** * When adding new insertions due to FK constraints, specify if * should get all columns for those new insertions, or just the minimal @@ -730,7 +731,7 @@ class SqlInsertBuilder( for (fk in table.foreignKeys) { - val targetTable = fk.targetTable + val targetTable = fk.targetTableId // if we have already generated the sql inserts for the target table more than 3 times and all columns are nullable, then skip it val maxIter = 3 // TODO: as a configurable parameter in EMConfig? @@ -999,7 +1000,7 @@ class SqlInsertBuilder( * @param tables to check * @param all is a complete set of tables with their fk */ - fun extractFkTable(tables: Set, all: MutableSet = mutableSetOf()): Set { + fun extractFkTable(tables: Set, all: MutableSet = mutableSetOf()): Set { tables.forEach { t -> if (!all.contains(t)) all.add(t) @@ -1011,13 +1012,10 @@ class SqlInsertBuilder( return all.toSet() } - private fun extractFkTable(tableName: String): Set { + private fun extractFkTable(tableId: TableId): Set { return tables.filter { t -> t.value.foreignKeys.any { f -> - f.targetTable.equals( - tableName, - ignoreCase = true - ) + f.targetTableId == tableId } }.keys } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt index b9c6d7c004..6a94623786 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/ForeignKey.kt @@ -9,5 +9,5 @@ data class ForeignKey( val sourceColumns: Set, - val targetTable: String + val targetTableId: TableId ) \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt index 23f8169de3..865b750f76 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt @@ -1,5 +1,11 @@ package org.evomaster.core.sql.schema +import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto + +/** + * MUST be kept immutable + */ data class TableId( /** @@ -48,6 +54,26 @@ data class TableId( } } + companion object{ + + fun fromDto(type: DatabaseType, dto: TableIdDto) : TableId{ + + val connectionId = null //TODO + + return when(type){ + DatabaseType.POSTGRES, DatabaseType.H2 -> + TableId(dto.name, connectionId, dto.catalog, dto.schema) + + DatabaseType.MYSQL, DatabaseType.MARIADB -> + TableId(dto.name, connectionId, null, dto.schema ?: dto.catalog) + + else -> + TableId(dto.name, connectionId, dto.catalog, dto.schema) + } + } + } + + fun getFullQualifyingTableName() : String{ if(openGroupName.isNullOrBlank()){ return name diff --git a/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt b/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt index 26226f65c3..5991ee92a9 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/structuralelement/individual/RestIndividualStructureTest.kt @@ -42,7 +42,7 @@ class RestIndividualStructureTest : StructuralElementBaseTest(){ unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = foo.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = foo.id) val bar = Table("Bar", setOf(barIdColumn, fkColumn), setOf(foreignKey)) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt index 06af4490f4..c268540a44 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt @@ -1,10 +1,6 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType -import org.evomaster.core.sql.schema.Column -import org.evomaster.core.sql.schema.ColumnDataType -import org.evomaster.core.sql.schema.ForeignKey -import org.evomaster.core.sql.schema.Table import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.enterprise.SampleType import org.evomaster.core.search.gene.* @@ -14,6 +10,7 @@ import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.* import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -415,7 +412,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -427,7 +424,7 @@ class DbActionUtilsTest { val action0 = SqlAction(table0, setOf(idColumn), insertId0, listOf(pkGeneTable0)) val insertId1 = 1002L - val fkGene = SqlForeignKeyGene("Id", insertId1, "Table0", false, insertId0) + val fkGene = SqlForeignKeyGene("Id", insertId1, TableId("Table0"), false, insertId0) val pkGeneTable1 = SqlPrimaryKeyGene("Id", "Table1", fkGene, insertId1) val action1 = SqlAction(table1, setOf(fkColumn), insertId1, listOf(pkGeneTable1)) @@ -459,7 +456,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -470,7 +467,7 @@ class DbActionUtilsTest { val action0 = SqlAction(table0, setOf(idColumn), insertId0, listOf(pkGeneTable0)) val insertId1 = 1002L - val fkGene = SqlForeignKeyGene("Id", insertId1, "Table0", false, insertId0) + val fkGene = SqlForeignKeyGene("Id", insertId1, TableId("Table0"), false, insertId0) val pkGeneTable1 = SqlPrimaryKeyGene("Id", "Table1", fkGene, insertId1) val action1 = SqlAction(table1, setOf(fkColumn), insertId1, listOf(pkGeneTable1)) @@ -501,7 +498,7 @@ class DbActionUtilsTest { unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) @@ -518,12 +515,12 @@ class DbActionUtilsTest { val insertId2 = 1003L - val fkGene0 = SqlForeignKeyGene("Id", insertId2, "Table0", false, insertId0) + val fkGene0 = SqlForeignKeyGene("Id", insertId2, TableId("Table0"), false, insertId0) val pkGene2 = SqlPrimaryKeyGene("Id", "Table1", fkGene0, insertId2) val action2 = SqlAction(table1, setOf(fkColumn), insertId2, listOf(pkGene2)) val insertId3 = 1003L - val fkGene1 = SqlForeignKeyGene("Id", insertId3, "Table0", false, insertId0) + val fkGene1 = SqlForeignKeyGene("Id", insertId3, TableId("Table0"), false, insertId0) val pkGene3 = SqlPrimaryKeyGene("Id", "Table1", fkGene1, insertId3) val action3 = SqlAction(table1, setOf(fkColumn), insertId3, listOf(pkGene3)) @@ -557,7 +554,7 @@ class DbActionUtilsTest { autoIncrement = false, unique = false, databaseType = DatabaseType.H2) - val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTable = table0.name) + val foreignKey = ForeignKey(sourceColumns = setOf(fkColumn), targetTableId = table0.id) val table1 = Table("Table1", setOf(fkColumn), setOf(foreignKey)) val set = listOf(table0, table0, table1) diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index bc1b658c74..d676f80a09 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -18,6 +18,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene; import org.evomaster.core.search.service.FitnessFunction; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.core.sql.schema.TableId; import org.junit.jupiter.api.Test; import java.util.Collections; @@ -139,7 +140,7 @@ public void testSteps() { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion("DB_DIRECT_INT_ENTITY", Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call diff --git a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt index 5209320bd7..ed5ecc319e 100644 --- a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt +++ b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt @@ -5,6 +5,7 @@ import org.evomaster.core.problem.rest.HttpVerb import org.evomaster.core.problem.rest.service.RestSampler import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene +import org.evomaster.core.sql.schema.TableId import org.evomaster.e2etests.utils.RestTestBase import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll @@ -94,7 +95,7 @@ class BasicEMTest : RestTestBase() { val injector = init(args) val sampler = injector.getInstance(RestSampler::class.java) - val dbactions = sampler.sampleSqlInsertion("X", setOf("*")) + val dbactions = sampler.sampleSqlInsertion(TableId("X"), setOf("*")) assertEquals(1, dbactions.size) diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index 24619b0bba..d056c769fe 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene; import org.evomaster.core.search.service.FitnessFunction; import org.evomaster.ci.utils.CIUtils; +import org.evomaster.core.sql.schema.TableId; import org.junit.jupiter.api.Test; import java.util.Collections; @@ -140,7 +141,7 @@ public void testSteps() { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion("DB_DIRECT_INT_ENTITY", Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call From ec435838ed0fa354865974b854e94e9fbd14a6a0 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 15 Jan 2025 15:15:12 +0100 Subject: [PATCH 18/50] more on TableId --- .../enterprise/EnterpriseIndividual.kt | 5 +-- .../evomaster/core/sql/DatabaseExecution.kt | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt index 94e518b14d..20906c1355 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt @@ -13,6 +13,7 @@ import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.tracer.TrackOperator +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import java.util.* @@ -391,8 +392,8 @@ abstract class EnterpriseIndividual( /** * @return a list table names which are used to insert data directly */ - open fun getInsertTableNames(): List{ - return sqlInitialization.filterNot { it.representExistingData }.map { it.table.name } + open fun getInsertTableNames(): List{ + return sqlInitialization.filterNot { it.representExistingData }.map { it.table.id } } override fun seeTopGenes(filter: ActionFilter): List { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index ba9863d954..0f38de640d 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -1,6 +1,7 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecutionsDto +import org.evomaster.core.sql.schema.TableId /** * When a test case is executed, and the SUT does access a SQL database, @@ -16,11 +17,11 @@ import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecut * This class MUST be immutable */ class DatabaseExecution( - val queriedData: Map>, - val updatedData: Map>, - val insertedData: Map>, - val failedWhere: Map>, - val deletedData: List, + val queriedData: Map>, + val updatedData: Map>, + val insertedData: Map>, + val failedWhere: Map>, + val deletedData: List, val numberOfSqlCommands: Int, val sqlParseFailureCount: Int, val executionInfo: List @@ -38,15 +39,15 @@ class DatabaseExecution( validateQuotes(deletedData) } - private fun validateQuotes(data: Map>) { - val quoted = data.keys.filter { it.startsWith("\"") } + private fun validateQuotes(data: Map>) { + val quoted = data.keys.filter { it.name.startsWith("\"") } if(quoted.isNotEmpty()) { throw IllegalArgumentException("Following table names are quoted: ${quoted.joinToString(", ")}") } } - private fun validateQuotes(data: List) { - val quoted = data.filter { it.startsWith("\"") } + private fun validateQuotes(data: List) { + val quoted = data.filter { it.name.startsWith("\"") } if(quoted.isNotEmpty()) { throw IllegalArgumentException("Following table names are quoted: ${quoted.joinToString(", ")}") } @@ -66,6 +67,8 @@ class DatabaseExecution( might depend on library used to analyze the SQL commands. */ + FIXME + return DatabaseExecution( cloneData(dealWithQuotes(dto?.queriedData)), cloneData(dealWithQuotes(dto?.updatedData)), @@ -92,11 +95,11 @@ class DatabaseExecution( return data } - private fun dealWithQuotes(data: Map>?): Map>? { + private fun dealWithQuotes(data: Map>?): Map>? { if(data == null) return null return data.entries.associate { - (if(it.key.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value + (if(it.key.name.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value } } @@ -124,8 +127,8 @@ class DatabaseExecution( return data.map { SqlExecutionInfo(it.sqlCommand, it.threwSqlExeception, it.executionTime) } } - private fun cloneData(data: Map>?): Map> { - val clone = mutableMapOf>() + private fun cloneData(data: Map>?): Map> { + val clone = mutableMapOf>() data?.keys?.forEach { clone[it] = data[it]!!.toSet() @@ -133,8 +136,8 @@ class DatabaseExecution( return clone } - private fun merge(current: MutableMap>, - toAdd: Map>) { + private fun merge(current: MutableMap>, + toAdd: Map>) { for (e in toAdd.entries) { val key = e.key From dbd2e903e471673b1b7e9ad3234c6d6e17393b8b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 17 Jan 2025 15:37:54 +0100 Subject: [PATCH 19/50] more refactoring --- .../core/output/service/TestSuiteWriter.kt | 15 ++-- .../api/service/ApiWsStructureMutator.kt | 44 +++++++----- .../enterprise/service/EnterpriseSampler.kt | 2 +- .../core/problem/rest/RestIndividual.kt | 5 +- .../problem/rest/resource/ResourceCluster.kt | 31 +++++---- .../resource/ResourceImpactOfIndividual.kt | 17 ++--- .../problem/rest/resource/RestResourceNode.kt | 5 +- .../dependency/ResourceRelatedToTable.kt | 68 +++++++++++-------- .../rest/service/ResourceDepManageService.kt | 23 +++++-- .../rest/service/ResourceManageService.kt | 3 +- .../service/ResourceRestStructureMutator.kt | 11 ++- .../util/inference/DeriveResourceBinding.kt | 5 +- .../problem/util/inference/SimpleDeriveR2T.kt | 42 +++++++++--- .../util/inference/model/ParamGeneBindMap.kt | 9 ++- .../org/evomaster/core/search/FitnessValue.kt | 3 +- .../evomaster/core/sql/DatabaseExecution.kt | 6 +- .../core/sql/SqlActionGeneBuilder.kt | 2 +- .../org/evomaster/core/sql/SqlActionUtils.kt | 18 ++--- .../evomaster/core/sql/SqlInsertBuilder.kt | 35 ++++++---- 19 files changed, 215 insertions(+), 129 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 7806630aba..553489aed6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -22,6 +22,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.sql.schema.Table import org.evomaster.core.sql.schema.TableId import org.evomaster.test.utils.EMTestUtils import org.evomaster.test.utils.SeleniumEMUtils @@ -213,14 +214,15 @@ class TestSuiteWriter { } val all = (sampler as EnterpriseSampler).extractFkTables(accessedTable) - val tableNamesInSchema = remoteController.getCachedSutInfo() - ?.sqlSchemaDto + val schema = remoteController.getCachedSutInfo()?.sqlSchemaDto + + val tableNamesInSchema = schema ?.tables - ?.map { SqlDtoUtils.getId(it) } + ?.map { TableId.fromDto(schema.databaseType, it.id) } ?.toSet() ?: setOf() - val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y.equals(x,true) } }.sorted() + val missingTables = all.filter { x -> tableNamesInSchema.none { y -> y == x } }.sortedBy { it.name } if(missingTables.isNotEmpty()){ /* Weird case... but actually seen it in familie-ba-sak, regarding table "task", which is in the migration @@ -232,7 +234,10 @@ class TestSuiteWriter { } val input = if(all.isEmpty()) "" - else all.filter { x -> tableNamesInSchema.any{y -> y.equals(x,true)} }.sorted().joinToString(",") { "\"$it\"" } + else all.filter { x -> tableNamesInSchema.any{y -> y == x} } + .map{it.getFullQualifyingTableName()} + .sorted() + .joinToString(",") { "\"$it\"" } return when { config.outputFormat.isJava() -> "Arrays.asList($input)" diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt index 7e399ea40c..38d15bbbd0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt @@ -25,6 +25,7 @@ import org.evomaster.core.search.service.mutator.StructureMutator import org.evomaster.core.sql.SqlAction import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.SqlInsertBuilder +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -258,7 +259,9 @@ abstract class ApiWsStructureMutator : StructureMutator() { * its phenotype (otherwise the fitness value would be meaningless). */ - val fw = evaluatedIndividual.fitness.getViewOfAggregatedFailedWhere() + val fw = evaluatedIndividual + .fitness + .getViewOfAggregatedFailedWhere() //TODO likely to remove/change once we ll support VIEWs .filter { sampler.canInsertInto(it.key) } @@ -288,7 +291,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { /** * Map of FAILED WHERE clauses. from table name key to column name values */ - fw: Map>, + fw: Map>, mutatedGenes: MutatedGeneSpecification?, sampler: ApiWsSampler ): MutableList>? { @@ -307,7 +310,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { ind: T, sampler: ApiWsSampler, mutatedGenes: MutatedGeneSpecification?, - fw: Map> + fw: Map> ): MutableList>? { /* because there might exist representExistingData in db actions which are in between rest actions, @@ -385,7 +388,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedSqlInsertions } - private fun handleDSE(sampler: ApiWsSampler, fw: Map>): MutableList> { + private fun handleDSE(sampler: ApiWsSampler, fw: Map>): MutableList> { /* TODO: DSE should be plugged in here */ return mutableListOf() } @@ -407,14 +410,17 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedMongoDbInsertions } - private fun findMissing(fw: Map>, dbactions: List): Map> { + private fun findMissing( + fw: Map>, + dbactions: List + ): Map> { return fw.filter { e -> //shouldn't have already an action adding such SQL data dbactions .filter { !it.representExistingData } .none { a -> - a.table.name.equals(e.key, ignoreCase = true) && e.value.all { c -> + a.table.id == e.key && e.value.all { c -> // either the selected column is already in existing action (c != "*" && a.selectedColumns.any { x -> x.name.equals(c, ignoreCase = true) @@ -440,25 +446,26 @@ abstract class ApiWsStructureMutator : StructureMutator() { note that if there is no any init sql, we randomly select one table to add. */ - val candidatesToMutate = - individual.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData } - val tables = candidatesToMutate.map { it.table.name }.run { - ifEmpty { getSqlInsertBuilder()!!.getTableNames() } - } + val candidatesToMutate = individual.seeInitializingActions() + .filterIsInstance() + .filterNot { it.representExistingData } + val tables = candidatesToMutate + .map { it.table.id } + .ifEmpty { getSqlInsertBuilder()!!.getTableNames() } - val table = randomness.choose(tables) - val total = tables.count { it == table } + val tableId = randomness.choose(tables) + val total = tables.count { it == tableId } if (total == 1 || randomness.nextBoolean()) { // add action val num = randomness.nextInt(1, max(1, getMaxSizeOfMutatingInitAction())) - val add = createInsertSqlAction(table, num) + val add = createInsertSqlAction(tableId, num) handleInitSqlAddition(individual, add, mutatedGenes) } else { // remove action val num = randomness.nextInt(1, max(1, min(total - 1, getMaxSizeOfMutatingInitAction()))) - val candidates = candidatesToMutate.filter { it.table.name == table } + val candidates = candidatesToMutate.filter { it.table.id == tableId } val remove = randomness.choose(candidates, num) handleInitSqlRemoval(individual, remove, mutatedGenes) @@ -520,7 +527,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { * @param name is the table name * @param num is a number of table with [name] to be added */ - fun createInsertSqlAction(name: String, num: Int): List> { + fun createInsertSqlAction(name: TableId, num: Int): List> { getSqlInsertBuilder() ?: throw IllegalStateException("attempt to create resource with SQL but the sqlBuilder is null") if (num <= 0) @@ -537,7 +544,10 @@ abstract class ApiWsStructureMutator : StructureMutator() { } val list = (0 until num) - .map { getSqlInsertBuilder()!!.createSqlInsertionAction(name,chosenColumns, mutableListOf(),true, extraConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) } + .map { + getSqlInsertBuilder()!! + .createSqlInsertionAction(name,chosenColumns, mutableListOf(),true, extraConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) + } .toMutableList() if (log.isTraceEnabled) { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index b9d81618b0..ee72836a4e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -85,7 +85,7 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { return action } - fun canInsertInto(tableName: String) : Boolean { + fun canInsertInto(tableName: TableId) : Boolean { //TODO might need to refactor/remove once we deal with VIEWs return sqlInsertBuilder?.isTable(tableName) ?: false } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt index d49eada1f7..34a409de99 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/RestIndividual.kt @@ -21,6 +21,7 @@ import org.evomaster.core.search.gene.* import org.evomaster.core.search.tracer.Traceable import org.evomaster.core.search.tracer.TraceableElementCopyFilter import org.evomaster.core.search.tracer.TrackOperator +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -402,8 +403,8 @@ class RestIndividual( } - override fun getInsertTableNames(): List { - return seeSqlDbActions().filterNot { it.representExistingData }.map { it.table.name } + override fun getInsertTableNames(): List { + return seeSqlDbActions().filterNot { it.representExistingData }.map { it.table.id } } override fun seeMainExecutableActions(): List { diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt index a2bc2bfe3a..953a0fafff 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt @@ -12,6 +12,7 @@ import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.util.inference.SimpleDeriveResourceBinding import org.evomaster.core.search.action.Action import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId /** * this class is to record the identified resources in the sut, i.e., based on the 'paths' of 'OpenAPI' @@ -29,13 +30,13 @@ class ResourceCluster { * key is table name * value is a list of existing data of PKs in DB */ - private val dataInDB : MutableMap> = mutableMapOf() + private val dataInDB : MutableMap> = mutableMapOf() /** * key is table name * value is the table */ - private val tables : MutableMap = mutableMapOf() + private val tables : MutableMap = mutableMapOf() /** @@ -126,10 +127,10 @@ class ResourceCluster { /** * @return existing rows of the table based on the specified [tableName] */ - fun getDataInDb(tableName: String) : MutableList?{ - val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) - ?: return null - return dataInDB.getValue(key) + fun getDataInDb(tableName: TableId) : MutableList?{ +// val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) +// ?: return null + return dataInDB.getValue(tableName) } /** @@ -154,7 +155,7 @@ class ResourceCluster { * if [doNotCreateDuplicatedAction], the returned list would be A and B * else the return list would be A, A and B. the additional A is created for B */ - fun createSqlAction(tables: List, + fun createSqlAction(tables: List, sqlInsertBuilder: SqlInsertBuilder, previous: List, doNotCreateDuplicatedAction: Boolean, @@ -164,15 +165,17 @@ class ResourceCluster { useExtraSqlDbConstraints: Boolean = false, enableSingleInsertionForTable : Boolean = false ) : MutableList{ - val sorted = SqlActionUtils.sortTable(tables.mapNotNull { getTableByName(it) }.run { if (doNotCreateDuplicatedAction) this.distinct() else this }) + val sorted = SqlActionUtils.sortTable( + tables.mapNotNull { this.tables[it] }.run { if (doNotCreateDuplicatedAction) this.distinct() else this } + ) val added = mutableListOf() val preTables = previous.filter { !isInsertion || !it.representExistingData }.map { it.table.name }.toMutableList() - if (!isInsertion && sorted.none { t-> (getDataInDb(t.name)?.size?: 0) > 0 }){ + if (!isInsertion && sorted.none { t-> (getDataInDb(t.id)?.size?: 0) > 0 }){ if (forceSynDataInDb) syncDataInDb(sqlInsertBuilder) - if (sorted.none { t-> (getDataInDb(t.name)?.size?: 0) > 0 }){ + if (sorted.none { t-> (getDataInDb(t.id)?.size?: 0) > 0 }){ return mutableListOf() } } @@ -180,19 +183,19 @@ class ResourceCluster { sorted.forEach { t-> if (!doNotCreateDuplicatedAction || preTables.none { p-> p.equals(t.name, ignoreCase = true) }){ val actions = if (!isInsertion){ - if ((getDataInDb(t.name)?.size ?: 0) == 0) { + if ((getDataInDb(t.id)?.size ?: 0) == 0) { null } else { - val candidates = getDataInDb(t.name)!! + val candidates = getDataInDb(t.id)!! val row = randomness.choose(candidates) - val a = sqlInsertBuilder.extractExistingByCols(t.name, row, useExtraSqlDbConstraints) + val a = sqlInsertBuilder.extractExistingByCols(t.id, row, useExtraSqlDbConstraints) if(a != null) listOf(a) else null } } else{ - sqlInsertBuilder.createSqlInsertionAction(t.name, useExtraSqlDbConstraints = useExtraSqlDbConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) + sqlInsertBuilder.createSqlInsertionAction(t.id, useExtraSqlDbConstraints = useExtraSqlDbConstraints, enableSingleInsertionForTable=enableSingleInsertionForTable) .onEach { a -> a.doInitialize(randomness) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt index c6af451554..f5ca92be15 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceImpactOfIndividual.kt @@ -8,6 +8,7 @@ import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfAction import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual import org.evomaster.core.search.impact.impactinfocollection.InitializationGroupedActionsImpacts import org.evomaster.core.search.impact.impactinfocollection.value.numeric.IntegerGeneImpact +import org.evomaster.core.sql.schema.TableId /** * created by manzhang on 2021/10/21 @@ -32,7 +33,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { * key - a name of table * value - impact */ - val sqlTableSizeImpact: MutableMap + val sqlTableSizeImpact: MutableMap /** * impact of changing size of any sql in the individual @@ -45,7 +46,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { dynamicMainActionImpacts: MutableList, impactsOfStructure: ActionStructureImpact = ActionStructureImpact("StructureSize"), resourceSizeImpact: MutableMap, - sqlTableImpact: MutableMap, + sqlTableImpact: MutableMap, anyResourceSizeImpact: IntegerGeneImpact, anySqlTableSizeImpact: IntegerGeneImpact @@ -63,9 +64,9 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { putIfAbsent(r, IntegerGeneImpact("size")) } } - sqlTableSizeImpact = mutableMapOf().apply { + sqlTableSizeImpact = mutableMapOf().apply { individual.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.forEach { d-> - putIfAbsent(d.table.name, IntegerGeneImpact("size")) + putIfAbsent(d.table.id, IntegerGeneImpact("size")) } } anyResourceSizeImpact = IntegerGeneImpact("anyResourceSizeImpact") @@ -84,7 +85,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { mutableMapOf().apply { putAll(resourceSizeImpact.map { it.key to it.value.copy() }) }, - mutableMapOf().apply { + mutableMapOf().apply { putAll(sqlTableSizeImpact.map { it.key to it.value.copy() }) }, anyResourceSizeImpact.copy(), @@ -104,7 +105,7 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { mutableMapOf().apply { putAll(resourceSizeImpact.map { it.key to it.value.clone() }) }, - mutableMapOf().apply { + mutableMapOf().apply { putAll(sqlTableSizeImpact.map { it.key to it.value.clone() }) }, anyResourceSizeImpact.clone(), @@ -130,8 +131,8 @@ class ResourceImpactOfIndividual : ImpactsOfIndividual { } } - val currentTs = current.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.name } - val previousTs = previous.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.name } + val currentTs = current.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.id } + val previousTs = previous.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }.map { it.table.id } var anySqlChange = false currentTs.toSet().forEach { cr -> val tImpact = sqlTableSizeImpact.getOrPut(cr){IntegerGeneImpact("size")} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt index 3ac9ca4a93..d6fb1e19fa 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -165,7 +166,7 @@ open class RestResourceNode( } return dbactions.filterNot { it.representExistingData }.flatMap { db-> - val exclude = related.flatMap { r-> r?.getRelatedColumn(db.table.name)?.toList()?:listOf() } + val exclude = related.flatMap { r-> r?.getRelatedColumn(db.table.id)?.toList()?:listOf() } db.seeGenesForInsertion(exclude) }.filter{it.isMutable() && it !is SqlForeignKeyGene && it !is SqlPrimaryKeyGene} } @@ -235,7 +236,7 @@ open class RestResourceNode( /** * @return related table for creating resource for [this] node with sql */ - fun getSqlCreationPoints() : List{ + fun getSqlCreationPoints() : List{ if (resourceToTable.confirmedSet.isNotEmpty()) return resourceToTable.confirmedSet.keys.toList() return resourceToTable.derivedMap.keys.toList() } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index bc9d513124..e771ddbdaa 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -6,6 +6,7 @@ import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo import org.evomaster.core.sql.SqlActionUtils +import org.evomaster.core.sql.schema.TableId /** * related info between resource and tables @@ -27,7 +28,7 @@ class ResourceRelatedToTable(val key: String) { * 2) segments, for instance there are two segments (i.e., /A and /B) regarding the path /A/{a}/B/{b}. for B, the input indicator is 0, for A, the input indicator is 1 * then detect whether there are tables related to C, A and B. */ - val derivedMap : MutableMap> = mutableMapOf() + val derivedMap : MutableMap> = mutableMapOf() /** * key is id of param @@ -44,7 +45,7 @@ class ResourceRelatedToTable(val key: String) { * to bind param of action regarding table, * we need to further detect whether the relationship between resource and table is direct */ - val confirmedSet : MutableMap = mutableMapOf() + val confirmedSet : MutableMap = mutableMapOf() /** * key is verb of the action @@ -89,21 +90,21 @@ class ResourceRelatedToTable(val key: String) { return doesUpdateParamTable } - fun getConfirmedDirectTables() : Set{ + fun getConfirmedDirectTables() : Set{ return derivedMap.keys .filter { t-> - confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } + confirmedSet.any { it.key == t && it.value } }.toHashSet() } fun findBestTableForParam( - tables: Set, + tables: Set, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false - ) : Pair, Double>? { + ) : Pair, Double>? { val map = simpleP2Table.derivedMap - .filter { d -> tables.any { t-> SqlActionUtils.isMatchingTableName(d.key, t) } } + .filter { d -> tables.any { t-> t == d.key } } .map { Pair(it.key, it.value.similarity) }.toMap() @@ -116,13 +117,13 @@ class ResourceRelatedToTable(val key: String) { * return Pair.first is name of table, Pair.second is column of Table */ fun getSimpleParamToSpecifiedTable( - table: String, + table: TableId, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false - ) : Pair? { + ) : Pair? { return simpleP2Table.derivedMap - .filter { table.equals(it.key, ignoreCase = true) } + .filter { table == it.key} .let { map-> if (map.isEmpty()) null else Pair(table, map.values.first().targetMatched) @@ -135,16 +136,20 @@ class ResourceRelatedToTable(val key: String) { * key is field name * value is a pair, the first of pair of related table and the second is similarity */ - fun findBestTableForParam(tables: Set, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : MutableMap, Double>>?{ + fun findBestTableForParam( + tables: Set, + bodyP2Table : BodyParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : MutableMap, Double>>?{ val fmap = bodyP2Table.fieldsMap .filter { - it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } + it.value.derivedMap.any { m-> tables.any { t-> t == m.key } } } if(fmap.isEmpty()) return null - val result : MutableMap, Double>> = mutableMapOf() + val result : MutableMap, Double>> = mutableMapOf() fmap.forEach { t, u -> - val related = u.derivedMap.filter { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) }} + val related = u.derivedMap.filter { m-> tables.any { t-> t == m.key }} if (related.isNotEmpty()){ val best = related.map { it.value.similarity }.maxOrNull()!! result.put(t, Pair(related.filter { it.value.similarity == best }.keys, best)) @@ -156,10 +161,14 @@ class ResourceRelatedToTable(val key: String) { /** * return Pair.first is name of table, key of the second is field, value is the column */ - fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair>? { + fun getBodyParamToSpecifiedTable( + table:TableId, + bodyP2Table : BodyParamRelatedToTable, + onlyConfirmedColumn : Boolean = false + ) : Pair>? { val fmap = bodyP2Table.fieldsMap .filter { - it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } + it.value.derivedMap.any { m->m.key == table } } if(fmap.isEmpty()) return null else{ @@ -172,15 +181,20 @@ class ResourceRelatedToTable(val key: String) { * * @param onlyConfirmedColumn will be used to only find the field that are confirmed based on feedback from evomaster driver */ - fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, fieldName : String, onlyConfirmedColumn : Boolean = false) : Pair>? { + fun getBodyParamToSpecifiedTable( + table:TableId, + bodyP2Table : BodyParamRelatedToTable, + fieldName : String, + onlyConfirmedColumn : Boolean = false + ) : Pair>? { val fmap = bodyP2Table.fieldsMap[fieldName]?: return null - fmap.derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}.let { + fmap.derivedMap.filter { it.key == table}.let { if (it.isEmpty()) return null else return Pair(table, Pair(fieldName, it.values.first().targetMatched)) } } - fun getTablesInDerivedMap() : Set = derivedMap.keys + fun getTablesInDerivedMap() : Set = derivedMap.keys fun getTablesInDerivedMap(segs : List) : Map{ return segs.map { input-> @@ -255,12 +269,12 @@ abstract class ParamRelatedToTable ( * key is table name * value is [MatchedInfo], [MatchInfo.targetMatched] is name of column of the table */ - val derivedMap : MutableMap = mutableMapOf() + val derivedMap : MutableMap = mutableMapOf() val confirmedColumn : MutableSet = mutableSetOf() - open fun getRelatedColumn(table: String) : Set? = - derivedMap.filterKeys { SqlActionUtils.isMatchingTableName(it, table)} + open fun getRelatedColumn(table: TableId) : Set? = + derivedMap.filterKeys { it == table} .values.firstOrNull() .run { if (this == null) null @@ -287,16 +301,16 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ val fieldsMap : MutableMap = mutableMapOf() - override fun getRelatedColumn(table: String) : Set? = + override fun getRelatedColumn(table: TableId) : Set? = fieldsMap.values .filter { - it.derivedMap.any { m-> SqlActionUtils.isMatchingTableName(m.key, table)} + it.derivedMap.any { m-> m.key == table} }.run { if (this.isEmpty()) return null else this.map { f-> - f.derivedMap.filterKeys { k -> SqlActionUtils.isMatchingTableName(k, table) } + f.derivedMap.filterKeys { k -> k == table } .asSequence().first().value.targetMatched }.toHashSet() } @@ -306,8 +320,8 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT */ class ParamFieldRelatedToTable(key: String) : ParamRelatedToTable(key){ - override fun getRelatedColumn(table: String) : Set? { - return derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)} + override fun getRelatedColumn(table: TableId) : Set? { + return derivedMap.filter { it.key == table} .let { if(it.isEmpty()) null else it.values.map {m-> m.targetMatched}.toSet() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt index 6816e974f0..c3272e5261 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt @@ -28,6 +28,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import java.nio.file.Files @@ -99,7 +100,7 @@ class ResourceDepManageService { } - private fun updateParamInfo(action: RestCallAction, tables: Map) { + private fun updateParamInfo(action: RestCallAction, tables: Map) { val r = rm.getResourceNodeFromCluster(action.path.toString()) // skip resource if it is ExcludedResourceNode if (r is ExcludedResourceNode) return @@ -222,8 +223,13 @@ class ResourceDepManageService { } } - private fun updateResourceToTable(action: RestCallAction, dto: SqlExecutionsDto, tables: Map, - addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateResourceToTable( + action: RestCallAction, + dto: SqlExecutionsDto, + tables: Map, + addedMap: MutableMap>, + removedMap: MutableMap> + ) { dto.insertedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { added -> updateResourceToTable(action, added, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) @@ -1001,7 +1007,10 @@ class ResourceDepManageService { */ fun unRelatedSQL(ind: RestIndividual, candidates: List?) : List{ val allrelated = getAllRelatedTables(ind) - return (candidates?:ind.seeInitializingActions().filterIsInstance().filterNot { it.representExistingData }).filterNot { allrelated.any { r-> r.equals(it.table.name, ignoreCase = true) } } + return (candidates ?: ind.seeInitializingActions() + .filterIsInstance() + .filterNot { it.representExistingData }) + .filterNot { allrelated.any { r-> r == it.table.id } } } fun identifyUnRelatedSqlTable(ind: RestIndividual, candidates: List?) : List{ @@ -1025,7 +1034,7 @@ class ResourceDepManageService { * @param probability represent a probability of using identified dependent tables, otherwise employ the tables which are * not part of the current dbInitialization */ - fun identifyRelatedSQL(ind: RestIndividual, probability: Double = 1.0): Set{ + fun identifyRelatedSQL(ind: RestIndividual, probability: Double = 1.0): Set{ val allrelated = getAllRelatedTables(ind) @@ -1046,7 +1055,7 @@ class ResourceDepManageService { } } - fun createDbActions(name : String, num : Int) : List>{ + fun createDbActions(name : TableId, num : Int) : List>{ rm.getSqlBuilder() ?:throw IllegalStateException("attempt to create resource with SQL but the sqlBuilder is null") if (num <= 0) throw IllegalArgumentException("invalid num (i.e.,$num) for creating resource") @@ -1128,7 +1137,7 @@ class ResourceDepManageService { ind.addInitializingDbActions(actions = added) } - private fun getAllRelatedTables(ind: RestIndividual) : Set{ + private fun getAllRelatedTables(ind: RestIndividual) : Set{ return ind.getResourceCalls().flatMap { c-> extractRelatedTablesForCall(c, withSql = c.is2POST).values.flatMap { it.map { g->g.tableName } }.toSet() }.toSet() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt index 6a3cfd4fe8..b50473fc35 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt @@ -16,6 +16,7 @@ import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -296,7 +297,7 @@ class ResourceManageService { } - private fun employSelect(tables : List) : Boolean{ + private fun employSelect(tables : List) : Boolean{ return randomness.nextBoolean(config.probOfSelectFromDatabase) && tables.any { cluster.getDataInDb(it)?.size?:0 >= config.minRowOfTable } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt index e800506957..2eed980f41 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt @@ -17,6 +17,7 @@ import org.evomaster.core.search.impact.impactinfocollection.value.numeric.Integ import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.sql.schema.TableId import kotlin.math.max import kotlin.math.min import org.slf4j.Logger @@ -177,7 +178,7 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { val candidates = if (doesApplyDependencyHeuristics()) dm.identifyRelatedSQL(ind) else - ind.seeInitializingActions().filterIsInstance().map { it.table.name }.toSet() // adding an unrelated table would waste budget, then we add existing ones + ind.seeInitializingActions().filterIsInstance().map { it.table.id }.toSet() // adding an unrelated table would waste budget, then we add existing ones val selectedAdded = if (config.enableAdaptiveResourceStructureMutation){ adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.toList(), targets) @@ -190,7 +191,13 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { } - private fun adaptiveSelectResource(evaluatedIndividual: EvaluatedIndividual?, bySQL: Boolean, candidates: List, targets: Set?): String{ + private fun adaptiveSelectResource( + evaluatedIndividual: EvaluatedIndividual?, + bySQL: Boolean, + candidates: List, + targets: Set? + ): TableId{ + evaluatedIndividual?: throw IllegalStateException("lack of impact with specified evaluated individual") targets?:throw IllegalStateException("targets must be specified if adaptive resource selection is applied") if (evaluatedIndividual.impactInfo == null || evaluatedIndividual.impactInfo !is ResourceImpactOfIndividual) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt index a6ae006177..bd6e0249e0 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/DeriveResourceBinding.kt @@ -7,17 +7,18 @@ import org.evomaster.core.problem.rest.resource.ParamInfo import org.evomaster.core.problem.rest.resource.RestResourceCalls import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.util.inference.model.ParamGeneBindMap +import org.evomaster.core.sql.schema.TableId /** * process inference regarding resource, which can be extended for the different inference technique */ interface DeriveResourceBinding { - fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map) + fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map) /** * derive relationship between a resource and a table */ - fun deriveResourceToTable(resourceNode : RestResourceNode, allTables : Map) + fun deriveResourceToTable(resourceNode : RestResourceNode, allTables : Map) fun generateRelatedTables(paramsInfo: List, calls: RestResourceCalls, sqlActions : List) : MutableMap>? = null diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index f1649a039e..665991cf83 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -17,6 +17,7 @@ import org.evomaster.core.problem.util.StringSimilarityComparator.stringSimilari import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.sql.SqlActionUtils.SCHEMA_TABLE_SEPARATOR +import org.evomaster.core.sql.schema.TableId import kotlin.math.max /** @@ -31,7 +32,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * i.e., the action under the resource may manipulate data from the table * e.g., /A/{a}/B{b}, the resource is related to two resource A and B */ - override fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map){ + override fun deriveResourceToTable(resourceCluster: MutableList, allTables : Map){ resourceCluster.forEach { r-> deriveResourceToTable(r, allTables) } @@ -40,7 +41,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { /** * derive a relationship between a resource and a table */ - override fun deriveResourceToTable(resourceNode: RestResourceNode, allTables : Map){ + override fun deriveResourceToTable(resourceNode: RestResourceNode, allTables : Map){ //1. derive resource to table //1.1 derive resource to tables based on segments @@ -52,14 +53,14 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { table name might contain schema, */ val matchedMap = allTables.keys.map { - Pair(it, calculateStringSimilarityScoreWithTableName(it, token)) + Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), token)) }.asSequence().sortedBy { e->e.second } if(matchedMap.lastOrNull()!= null && matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(seg, it.first, similarity = it.second, inputIndicator = sindex, outputIndicator = 0)) + }.add(MatchedInfo(seg, it.first.getFullQualifyingTableName(), similarity = it.second, inputIndicator = sindex, outputIndicator = 0)) } return@stop } @@ -138,13 +139,13 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return score } - fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ + fun deriveParamsToTable(mapParamInfo : Map, r: RestResourceNode, allTables : Map){ mapParamInfo.forEach { (paramId, paramInfo) -> deriveParamsToTable(paramId, paramInfo, r, allTables) } } - fun deriveParamsToTable(paramId : String, paramInfo : ParamInfo, r: RestResourceNode, allTables : Map){ + fun deriveParamsToTable(paramId : String, paramInfo : ParamInfo, r: RestResourceNode, allTables : Map){ val inputIndicator = paramInfo.segmentLevel @@ -161,7 +162,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { r.resourceToTable.derivedMap.filter { it.value.any { m-> m.input == (ParamUtil.getObjectGene(paramInfo.referParam.gene))!!.refType!!} }.keys.toHashSet() } else null - if(tables == null || tables.isEmpty()) tables = relatedTables + if(tables.isNullOrEmpty()) tables = relatedTables created = deriveRelatedTable(r, paramId, paramInfo, tables, isBodyParam, inputIndicator, alltables = allTables ) } @@ -170,7 +171,15 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } } - fun deriveRelatedTable(r : RestResourceNode, paramId: String, paramInfo: ParamInfo, relatedToTables: Set, isBodyParam : Boolean, inputIndicator: Int, alltables : Map) : Boolean{ + fun deriveRelatedTable( + r : RestResourceNode, + paramId: String, + paramInfo: ParamInfo, + relatedToTables: Set, + isBodyParam : Boolean, + inputIndicator: Int, + alltables : Map + ) : Boolean{ if(isBodyParam){ val pToTable = BodyParamRelatedToTable(paramId, paramInfo.referParam) ParamUtil.getObjectGene(paramInfo.referParam.gene)?.fields?.forEach { f-> @@ -205,13 +214,24 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * * Note that same parameter (e.g., id) may related to multiple related tables. */ - private fun deriveParamWithTable(paramName : String, candidateTables : Set, pToTable : MutableMap, inputlevel: Int, tables : Map){ + private fun deriveParamWithTable( + paramName : String, + candidateTables : Set, + pToTable : MutableMap, + inputlevel: Int, + tables : Map){ candidateTables.forEach { tableName -> deriveParamWithTable(paramName, tableName, pToTable, inputlevel, tables) } } - private fun deriveParamWithTable(paramName : String, tableName: String, pToTable : MutableMap, inputlevel: Int, tables : Map){ + private fun deriveParamWithTable( + paramName : String, + tableName: String, + pToTable : MutableMap, + inputlevel: Int, + tables : Map + ){ /* paramName might be \w+id or \w+name, in this case, we compare paramName with table name of current or referred table + column name */ @@ -307,7 +327,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { } - private fun getBindMap(paramId: String, pToTable : ParamRelatedToTable, tables : Set, resourceToTable: ResourceRelatedToTable, result : MutableList) : Boolean{ + private fun getBindMap(paramId: String, pToTable : ParamRelatedToTable, tables : Set, resourceToTable: ResourceRelatedToTable, result : MutableList) : Boolean{ if(pToTable is SimpleParamRelatedToTable){ resourceToTable.findBestTableForParam(tables, pToTable)?.let {pair-> val target = pair.first.toList()[(0..(pair.first.size-1)).shuffled().first()] diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt index 617a4fb4ca..9213363530 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/model/ParamGeneBindMap.kt @@ -1,9 +1,16 @@ package org.evomaster.core.problem.util.inference.model +import org.evomaster.core.sql.schema.TableId + /** * [isElementOfParam] means the gene of Param is one type of ObjectGene, MapGene, ListGene */ -class ParamGeneBindMap(val paramId : String, val isElementOfParam: Boolean, val targetToBind : String, val tableName: String, val column: String){ +class ParamGeneBindMap( + val paramId : String, + val isElementOfParam: Boolean, + val targetToBind : String, + val tableName: TableId, + val column: String){ fun equalWith(p: ParamGeneBindMap) : Boolean{ return paramId == p.paramId && isElementOfParam == p.isElementOfParam && targetToBind == p.targetToBind && tableName == p.tableName && column == p.column diff --git a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt index ee737c8876..ac64d7f0e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/FitnessValue.kt @@ -11,6 +11,7 @@ import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceRequest import org.evomaster.core.search.service.IdMapper import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory import kotlin.math.max @@ -81,7 +82,7 @@ class FitnessValue( * When SUT does SQL commands using WHERE, keep track of when those "fails" (ie evaluate * to false), in particular the tables and columns in them involved */ - private val aggregatedFailedWhere: MutableMap> = mutableMapOf() + private val aggregatedFailedWhere: MutableMap> = mutableMapOf() /** * When SUT does MONGO commands using FIND, keep track of when those "fails" (ie evaluate diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index 0f38de640d..29c0c29f92 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -83,10 +83,10 @@ class DatabaseExecution( fun mergeData( executions: Collection, - extractor: (DatabaseExecution) -> Map> - ): Map> { + extractor: (DatabaseExecution) -> Map> + ): Map> { - val data: MutableMap> = mutableMapOf() + val data: MutableMap> = mutableMapOf() for (ex in executions) { merge(data, extractor.invoke(ex)) diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt index d65c7eb55e..c95adcafae 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionGeneBuilder.kt @@ -388,7 +388,7 @@ class SqlActionGeneBuilder { } if (column.primaryKey) { - gene = SqlPrimaryKeyGene(column.name, table.name, gene, id) + gene = SqlPrimaryKeyGene(column.name, table.id, gene, id) } if (column.nullable && fk == null) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt index 769e693bd4..a14bef1f70 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt @@ -9,6 +9,7 @@ import org.evomaster.core.search.service.Randomness import org.slf4j.Logger import org.slf4j.LoggerFactory import org.evomaster.core.sql.schema.Table +import org.evomaster.core.sql.schema.TableId object SqlActionUtils { @@ -484,7 +485,7 @@ object SqlActionUtils { throw IllegalStateException("invalid insertion, there exists invalid fk at $index") val pks = sqlActions.subList(0, index).flatMap { it.seeTopGenes() }.filterIsInstance() fks.filter { !it.nullable && !it.isBound() || pks.none { p->p.uniqueId == it.uniqueIdOfPrimaryKey }}.forEach {fk-> - val found = pks.find { pk -> pk.tableName.equals(fk.targetTable, ignoreCase = true) } + val found = pks.find { pk -> pk.tableName == fk.targetTable } ?: throw IllegalStateException("fail to target table ${fk.targetTable} for the fk ${fk.name}") fk.uniqueIdOfPrimaryKey = found.uniqueId } @@ -496,14 +497,13 @@ object SqlActionUtils { /** * @return a list of dbactions from [sqlActions] whose related table is [tableName] */ - fun findDbActionsByTableName(sqlActions: List, tableName : String) : List{ - return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) - || it.table.name.endsWith(".$tableName", true)} + fun findDbActionsByTableName(sqlActions: List, tableName : TableId) : List{ + return sqlActions.filter { it.table.id == tableName } } /** - * Are the 2 names matching? This ignore case. + * Are the 2 names matching? This ignores case. * The first [fullName] is a full qualifying name, including schema. * The second [name] "might" be simple, or full qualifying. */ @@ -527,18 +527,18 @@ object SqlActionUtils { * 2) keys having full names (eg, including schemas and possibly catalog) whereas input only having the name. * this latter case is not a problem if names are unique */ - fun getTableKey(keys: Set, tableName: String) : String?{ + fun getTableKey(keys: Set, tableName: String) : TableId?{ /* * SQL is not case sensitivity, table/column must ignore case sensitivity. - * No, this is not really true... + * FIXME: No, this is not really true... * Usually, names are lowered-cased by the DB, unless quoted in "": * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL. * */ - val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) } + val tableNameKey = keys.find { tableName.equals(it.getFullQualifyingTableName(), ignoreCase = true) } if (!tableName.contains(".") && tableNameKey == null){ //input name might be without schema, so check for partial match - val candidates = keys.filter { it.endsWith(".${tableName}", true) } + val candidates = keys.filter { it.name.equals(tableName, true) } if(candidates.size > 1){ throw IllegalArgumentException("Ambiguity." + " More than one candidate of table called $tableName." + diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 6653f99dea..97a60366f3 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -182,7 +182,6 @@ class SqlInsertBuilder( for (fk in tableDto.foreignKeys) { val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable) - FIXME if(tableKey == null || tableToColumns[tableKey] == null) { throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}") @@ -194,7 +193,9 @@ class SqlInsertBuilder( // val c = targetTable.find { it.name.equals(cname, ignoreCase = true) } // ?: throw IllegalArgumentException("Issue in foreign key: table ${f.targetTable} does not have a column called $cname") - val c = tableToColumns[SqlDtoUtils.getId(tableDto)]!!.find { it.name.equals(cname, ignoreCase = true) } + val id = TableId.fromDto(databaseType, tableDto.id) + + val c = tableToColumns[id]!!.find { it.name.equals(cname, ignoreCase = true) } ?: throw IllegalArgumentException("Issue in foreign key: table ${tableDto.id.name} does not have a column called $cname") sourceColumns.add(c) @@ -632,13 +633,17 @@ class SqlInsertBuilder( * * quotes "" can be used to force case-sensitivity */ - fun isTable(tableName: String) = SqlActionUtils.getTableKey(tables.keys, tableName) != null + fun isTable(tableName: TableId) = + //SqlActionUtils.getTableKey(tables.keys, tableName) != null + tables.contains(tableName) - private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ - val key = SqlActionUtils.getTableKey(map.keys, tableName) - return map[key] - } + +// private fun getValueByTableNameKey(map: Map, tableName: String) : T?{ +// +// val key = SqlActionUtils.getTableKey(map.keys, tableName) +// return map[key] +// } @@ -715,7 +720,7 @@ class SqlInsertBuilder( val table = getTable(tableName, useExtraSqlDbConstraints) val takeAll = columnNames.contains("*") - validateColumnNamesInput(takeAll, columnNames, table, tableName) + validateColumnNamesInput(takeAll, columnNames, table, tableName.getFullQualifyingTableName()) // filter only columns that will generate value for val selectedColumns = table.columns @@ -735,7 +740,7 @@ class SqlInsertBuilder( // if we have already generated the sql inserts for the target table more than 3 times and all columns are nullable, then skip it val maxIter = 3 // TODO: as a configurable parameter in EMConfig? - val visitedTableCount = history.count { it.equals(targetTable, ignoreCase = true) } + val visitedTableCount = history.count { it == targetTable } if (visitedTableCount >= maxIter && fk.sourceColumns.all { it.nullable }) { continue } @@ -844,7 +849,7 @@ class SqlInsertBuilder( val pkName = pks[i].name val inQuotes = pks[i].type.shouldBePrintedInQuotes || pks[i].dimension > 0 val data = ImmutableDataHolderGene(pkName, r.columnData[i], inQuotes) - val pk = SqlPrimaryKeyGene(pkName, table.name, data, id) + val pk = SqlPrimaryKeyGene(pkName, table.id, data, id) genes.add(pk) } @@ -872,7 +877,7 @@ class SqlInsertBuilder( * */ fun extractExistingByCols( - tableName: String, + tableName: TableId, pkValues: DataRowDto, useExtraSqlDbConstraints: Boolean, columnIds: List = mutableListOf() @@ -942,7 +947,7 @@ class SqlInsertBuilder( val gene = if (cols[i].primaryKey) { SqlPrimaryKeyGene( colName, - table.name, + table.id, ImmutableDataHolderGene(colName, row.columnData[i], inQuotes), id ) @@ -963,7 +968,7 @@ class SqlInsertBuilder( /** * get existing pks in db */ - fun extractExistingPKs(dataInDB: MutableMap>) { + fun extractExistingPKs(dataInDB: MutableMap>) { if (dbExecutor == null) { throw IllegalStateException("No Database Executor registered for this object") @@ -980,7 +985,7 @@ class SqlInsertBuilder( val result: QueryResultDto = dbExecutor.executeDatabaseCommandAndGetQueryResults(dto) ?: continue - dataInDB.getOrPut(table.name) { result.rows.map { it }.toMutableList() } + dataInDB.getOrPut(table.id) { result.rows.map { it }.toMutableList() } } } @@ -988,7 +993,7 @@ class SqlInsertBuilder( /** * get table info */ - fun extractExistingTables(tablesMap: MutableMap? = null) { + fun extractExistingTables(tablesMap: MutableMap? = null) { if (tablesMap != null) { tablesMap.clear() tablesMap.putAll(tables) From 3483d4ca49300bc9de9c05a2ed24755eb3f31e69 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 17 Jan 2025 23:17:03 +0100 Subject: [PATCH 20/50] yet on the journey to refactor TableId --- .../core/problem/api/service/ApiWsFitness.kt | 2 - .../enterprise/service/EnterpriseFitness.kt | 13 ++- .../problem/rest/resource/RestResourceNode.kt | 5 +- .../dependency/ResourceRelatedToResource.kt | 8 +- .../dependency/ResourceRelatedToTable.kt | 38 ++++---- .../rest/service/ResourceDepManageService.kt | 86 +++++++++++-------- .../rest/service/ResourceManageService.kt | 5 +- .../service/ResourceRestStructureMutator.kt | 10 ++- .../problem/util/inference/SimpleDeriveR2T.kt | 8 +- .../evomaster/core/sql/DatabaseExecution.kt | 75 ++++++++++------ .../evomaster/core/sql/SqlInsertBuilder.kt | 2 +- 11 files changed, 157 insertions(+), 95 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt index 705d26f644..3419e52d80 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsFitness.kt @@ -30,8 +30,6 @@ abstract class ApiWsFitness : EnterpriseFitness() where T : Individual { @Inject protected lateinit var writer: TestSuiteWriter - @Inject - protected lateinit var sampler: Sampler lateinit var infoDto: SutInfoDto diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index 92a3ea75cf..b7e5e1b10c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -44,6 +44,8 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual @Inject protected lateinit var searchTimeController: SearchTimeController + @Inject + protected lateinit var sampler: EnterpriseSampler /** * @param allSqlActions specified the db actions to be executed @@ -255,10 +257,13 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual if (configuration.extractSqlExecutionInfo) { for (i in 0 until dto.extraHeuristics.size) { val extra = dto.extraHeuristics[i] - val databaseExecution = DatabaseExecution.fromDto(extra.sqlSqlExecutionsDto) - fv.setDatabaseExecution(i, databaseExecution) - if (databaseExecution.sqlParseFailureCount>0) { - statistics.reportSqlParsingFailures(databaseExecution.sqlParseFailureCount) + val sdto = extra.sqlSqlExecutionsDto + if(sdto != null) { + val databaseExecution = DatabaseExecution.fromDto(sdto, sampler.sqlInsertBuilder!!.getTableNames()) + fv.setDatabaseExecution(i, databaseExecution) + if (databaseExecution.sqlParseFailureCount > 0) { + statistics.reportSqlParsingFailures(databaseExecution.sqlParseFailureCount) + } } } fv.aggregateDatabaseData() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt index d6fb1e19fa..da7033b32c 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt @@ -19,6 +19,7 @@ import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -666,7 +667,9 @@ open class RestResourceNode( /** * @return derived tables */ - fun getDerivedTables() : Set = resourceToTable.derivedMap.flatMap { it.value.map { m->m.targetMatched } }.toHashSet() + fun getDerivedTables(all: Set) : Set = resourceToTable.derivedMap + .flatMap { it.value.mapNotNull { m-> SqlActionUtils.getTableKey(all, m.targetMatched) } } + .toHashSet() /** * @return is any POST, GET, PATCH, DELETE, PUT action? diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt index a8382f695d..03f030d35d 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.rest.resource.dependency import org.evomaster.core.problem.rest.RestPath +import org.evomaster.core.sql.schema.TableId /** * @param path is a list of path(s), which can be parsed with [RelatedTo.parseMultipleKey] @@ -49,8 +50,11 @@ open class ResourceRelatedToResources( * Note that related table might be derived based on token parser, not confirmed regarding evomaster driver. * [confirmedSet] is used to represent whether the mutual relation is confirmed. */ -class MutualResourcesRelations(mutualResources: List, probability: Double, var referredTables : MutableSet = mutableSetOf()) - : ResourceRelatedToResources(mutualResources, mutualResources.toMutableList(), probability, ""){ +class MutualResourcesRelations( + mutualResources: List, + probability: Double, + var referredTables : MutableSet = mutableSetOf() +) : ResourceRelatedToResources(mutualResources, mutualResources.toMutableList(), probability, ""){ override fun getName(): String { return "MutualRelations:${notateKey()}" diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt index e771ddbdaa..de77cd5b7c 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt @@ -5,6 +5,7 @@ import org.evomaster.core.sql.SQLKey import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.util.inference.model.MatchedInfo +import org.evomaster.core.sql.DatabaseExecution import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId @@ -17,7 +18,7 @@ class ResourceRelatedToTable(val key: String) { companion object { private const val FROM_EVO_DTO = "____FROM_DTO____" - fun generateFromDtoMatchedInfo(table: String) : MatchedInfo = MatchedInfo(FROM_EVO_DTO, table, 1.0, -1, -1) + fun generateFromDtoMatchedInfo(table: TableId) : MatchedInfo = MatchedInfo(FROM_EVO_DTO, table.getFullQualifyingTableName(), 1.0, -1, -1) } /** * key is table name @@ -58,16 +59,17 @@ class ResourceRelatedToTable(val key: String) { private val actionToTables : MutableMap> = mutableMapOf() - fun updateActionRelatedToTable(verb : String, dto: SqlExecutionsDto, existingTables : Set) : Boolean{ + fun updateActionRelatedToTable(verb : String, de: DatabaseExecution, existingTables : Set) : Boolean{ - val tables = mutableListOf() - .plus(dto.deletedData) - .plus(dto.updatedData.keys) - .plus(dto.insertedData.keys) - .plus(dto.queriedData.keys) - .filter { x -> - existingTables.any { x.equals(it, true) } - }.toHashSet() + val tables = mutableListOf() + .plus(de.deletedData) + .plus(de.updatedData.keys) + .plus(de.insertedData.keys) + .plus(de.queriedData.keys) +// .filter { x -> +// existingTables.any { x.equals(it, true) } +// } + .toHashSet() if (tables.isEmpty()) return false @@ -82,10 +84,10 @@ class ResourceRelatedToTable(val key: String) { actionToTables[verb]!!.add(access) } - access.updateTableWithFields(dto.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) - access.updateTableWithFields(dto.insertedData, SQLKey.INSERT) - access.updateTableWithFields(dto.queriedData, SQLKey.SELECT) - access.updateTableWithFields(dto.updatedData, SQLKey.UPDATE) + access.updateTableWithFields(de.deletedData.associateWith { mutableSetOf() }, SQLKey.DELETE) + access.updateTableWithFields(de.insertedData, SQLKey.INSERT) + access.updateTableWithFields(de.queriedData, SQLKey.SELECT) + access.updateTableWithFields(de.updatedData, SQLKey.UPDATE) return doesUpdateParamTable } @@ -222,7 +224,7 @@ class ActionRelatedToTable( * key is table name * value is how it accessed by dbaction */ - val tableWithFields: MutableMap> = mutableMapOf() + val tableWithFields: MutableMap> = mutableMapOf() ) { @@ -230,7 +232,7 @@ class ActionRelatedToTable( * key of result is table name * value of result is a set of manipulated columns */ - fun updateTableWithFields(results : Map>, method: SQLKey) { + fun updateTableWithFields(results : Map>, method: SQLKey) { var doesUpdateTarget = false results.forEach { t, u -> doesUpdateTarget = doesUpdateTarget || tableWithFields.containsKey(t) @@ -245,7 +247,7 @@ class ActionRelatedToTable( } } - fun doesSubsume(tables : Set, subsumeThis : Boolean) : Boolean{ + fun doesSubsume(tables : Set, subsumeThis : Boolean) : Boolean{ return if(subsumeThis) tables.toHashSet().containsAll(tableWithFields.keys) else tableWithFields.keys.containsAll(tables) } @@ -255,7 +257,7 @@ class ActionRelatedToTable( * @property table related table * @property field what fields are assess by the sql command */ - class AccessTable(val method : SQLKey, val table : String, val field : MutableSet) + class AccessTable(val method : SQLKey, val table : TableId, val field : MutableSet) } /** diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt index c3272e5261..48c420a02b 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceDepManageService.kt @@ -28,6 +28,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.sql.DatabaseExecution import org.evomaster.core.sql.schema.TableId import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -83,16 +84,18 @@ class ResourceDepManageService { /* TODO how to decide to remove relationship between resource and table */ - val addedMap = mutableMapOf>() - val removedMap = mutableMapOf>() + val addedMap = mutableMapOf>() + val removedMap = mutableMapOf>() restIndividual.seeMainExecutableActions().forEachIndexed { index, action -> if (config.doesApplyNameMatching) updateParamInfo(action as RestCallAction, tables) // size of extraHeuristics might be less than size of action due to failure of handling rest action if (index < dto.extraHeuristics.size) { val dbDto = dto.extraHeuristics[index].sqlSqlExecutionsDto - if (dbDto != null) - updateResourceToTable(action as RestCallAction, dbDto, tables, addedMap, removedMap) + if (dbDto != null) { + val de = DatabaseExecution.fromDto(dbDto, tables.keys) + updateResourceToTable(action, de, tables, addedMap, removedMap) + } } } if (addedMap.isNotEmpty() || removedMap.isNotEmpty()) @@ -114,12 +117,20 @@ class ResourceDepManageService { /** * TODO remove false-derived dependencies based on feedback from evomaster driver */ - private fun updateDependencyOnceResourceTableUpdate(addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateDependencyOnceResourceTableUpdate( + addedMap: MutableMap>, + removedMap: MutableMap> + ) { val groupTable = addedMap.flatMap { it.value }.toHashSet() groupTable.forEach { table -> val newRelatedResource = addedMap.filter { it.value.contains(table) }.keys - val previousResourcesWithTable = dependencies.values.flatMap { l -> l.filter { d->d is MutualResourcesRelations && d.referredTables.contains(table) }.flatMap { it.targets }}.toHashSet() + val previousResourcesWithTable = dependencies.values + .flatMap { + l -> l + .filter { d->d is MutualResourcesRelations && d.referredTables.contains(table) } + .flatMap { it.targets } + }.toHashSet() var find = false dependencies.values.forEach { rlist -> @@ -170,8 +181,14 @@ class ResourceDepManageService { } } - private fun updateResourceToTable(action: RestCallAction, updated: Map>, matchedWithVerb: Boolean, tables: Map, - addedMap: MutableMap>, removedMap: MutableMap>) { + private fun updateResourceToTable( + action: RestCallAction, + updated: Map>, + matchedWithVerb: Boolean, + tables: Map, + addedMap: MutableMap>, + removedMap: MutableMap> + ) { val ar = rm.getResourceNodeFromCluster(action.path.toString()) val rToTable = ar.resourceToTable @@ -179,7 +196,7 @@ class ResourceDepManageService { val derivedTables = rToTable.getTablesInDerivedMap() updated.forEach { (t, u) -> - if (derivedTables.any { it.equals(t, ignoreCase = true) }) { + if (derivedTables.any { it == t }) { if (action.parameters.isNotEmpty() && u.isNotEmpty() && u.none { it == "*" }) { action.parameters.forEach { p -> val paramId = ar.getParamId(action.parameters, p) @@ -191,8 +208,8 @@ class ResourceDepManageService { } } } else { - val matchedInfo = ResourceRelatedToTable.generateFromDtoMatchedInfo(t.toLowerCase()) - ar.resourceToTable.derivedMap.put(t, mutableListOf(matchedInfo)) + val matchedInfo = ResourceRelatedToTable.generateFromDtoMatchedInfo(t) + ar.resourceToTable.derivedMap[t] = mutableListOf(matchedInfo) action.parameters.forEach { p -> val paramId = ar.getParamId(action.parameters, p) val paramInfo = ar.paramsInfo[paramId].run { @@ -204,13 +221,12 @@ class ResourceDepManageService { val hasMatchedParam = SimpleDeriveResourceBinding.deriveRelatedTable(ar, paramId, paramInfo, mutableSetOf(t), p is BodyParam, -1, alltables = tables) ar.resourceToTable.paramToTable[paramId]?.let { paramToTable -> paramToTable.getRelatedColumn(t)?.apply { - paramToTable.confirmedColumn.addAll(this.intersect(u.filter { it != "*" })) + paramToTable.confirmedColumn.addAll(this.intersect(u.filter { it != "*" }.toSet())) } } } addedMap.getOrPut(ar.getName()) { mutableSetOf() }.add(t) - } rToTable.confirmedSet.getOrPut(t) { true } @@ -225,29 +241,25 @@ class ResourceDepManageService { private fun updateResourceToTable( action: RestCallAction, - dto: SqlExecutionsDto, + de: DatabaseExecution, tables: Map, - addedMap: MutableMap>, - removedMap: MutableMap> + addedMap: MutableMap>, + removedMap: MutableMap> ) { - dto.insertedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { added -> - updateResourceToTable(action, added, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) - } - - dto.updatedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { updated -> - updateResourceToTable(action, updated, (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) - } + updateResourceToTable(action, de.insertedData, (action.verb == HttpVerb.POST || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.updatedData, (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.deletedData.map { Pair(it, mutableSetOf()) }.toMap(), (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + updateResourceToTable(action, de.queriedData, true, tables, addedMap, removedMap) - dto.deletedData.filter { u -> tables.any { it.key.toLowerCase() == u } }.let { del -> - updateResourceToTable(action, del.map { Pair(it, mutableSetOf()) }.toMap(), (action.verb == HttpVerb.PATCH || action.verb == HttpVerb.PUT), tables, addedMap, removedMap) + val relatedTable = rm.getResourceNodeFromCluster(action.path.toString()) + .resourceToTable - } - dto.queriedData.filter { u -> tables.any { it.key.toLowerCase() == u.key } }.let { get -> - updateResourceToTable(action, get, true, tables, addedMap, removedMap) - } - - rm.getResourceNodeFromCluster(action.path.toString()).resourceToTable.updateActionRelatedToTable(action.verb.toString(), dto, tables.keys) + relatedTable.updateActionRelatedToTable( + action.verb.toString(), + de, + tables.keys + ) } /************************ derive dependency using parser ***********************************/ @@ -257,7 +269,11 @@ class ResourceDepManageService { */ fun initDependencyBasedOnDerivedTables(resourceCluster: ResourceCluster) { resourceCluster.getTableInfo().keys.forEach { table -> - val mutualResources = resourceCluster.getCluster().values.filter { r -> r.getDerivedTables().any { e -> e.equals(table, ignoreCase = true) } }.map { it.getName() }.toList() + val mutualResources = resourceCluster.getCluster().values + .filter { r -> r.getDerivedTables(rm.sqlInsertBuilder!!.getTableNames()).any { e -> e == table } } + .map { it.getName() } + .toList() + if (mutualResources.isNotEmpty() && mutualResources.size > 1) { val mutualRelation = MutualResourcesRelations(mutualResources, StringSimilarityComparator.SimilarityThreshold, mutableSetOf(table)) @@ -267,7 +283,7 @@ class ResourceDepManageService { if (it == null) relations.add(mutualRelation) else - (it as MutualResourcesRelations).referredTables.add(table.toLowerCase()) + (it as MutualResourcesRelations).referredTables.add(table) } } } @@ -1040,7 +1056,7 @@ class ResourceDepManageService { if (allrelated.isNotEmpty() && randomness.nextBoolean(probability)){ val notincluded = allrelated.filterNot { - ind.seeInitializingActions().filterIsInstance().any { d-> it.equals(d.table.name, ignoreCase = true) } + ind.seeInitializingActions().filterIsInstance().any { d-> it == d.table.id } } //prioritize notincluded related ones with a probability 0.8 return if (notincluded.isNotEmpty() && randomness.nextBoolean(0.8)){ @@ -1048,7 +1064,7 @@ class ResourceDepManageService { }else allrelated }else{ val left = rm.getTableInfo().keys.filterNot { - ind.seeInitializingActions().filterIsInstance().any { d-> it.equals(d.table.name, ignoreCase = true) } + ind.seeInitializingActions().filterIsInstance().any { d-> it == d.table.id } } return if (left.isNotEmpty() && randomness.nextBoolean()) left.toSet() else rm.getTableInfo().keys diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt index b50473fc35..92552c0fec 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceManageService.kt @@ -53,7 +53,8 @@ class ResourceManageService { private val excludedCluster : MutableMap = mutableMapOf() - private var sqlInsertBuilder : SqlInsertBuilder? = null + var sqlInsertBuilder : SqlInsertBuilder? = null + private set /** * init resource nodes based on [actionCluster] and [sqlInsertBuilder] @@ -185,7 +186,7 @@ class ResourceManageService { return } - var employSQL = config.shouldGenerateSqlData() && hasDBHandler() && ar.getDerivedTables().isNotEmpty() + var employSQL = config.shouldGenerateSqlData() && hasDBHandler() && ar.getDerivedTables(sqlInsertBuilder!!.getTableNames()).isNotEmpty() && (forceSQLInsert || randomness.nextBoolean(config.probOfApplySQLActionToCreateResources)) var candidate = template diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt index 2eed980f41..0c82c33f3c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestStructureMutator.kt @@ -17,6 +17,7 @@ import org.evomaster.core.search.impact.impactinfocollection.value.numeric.Integ import org.evomaster.core.search.service.mutator.MutatedGeneSpecification import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.sql.SqlActionUtils import org.evomaster.core.sql.schema.TableId import kotlin.math.max import kotlin.math.min @@ -181,7 +182,9 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { ind.seeInitializingActions().filterIsInstance().map { it.table.id }.toSet() // adding an unrelated table would waste budget, then we add existing ones val selectedAdded = if (config.enableAdaptiveResourceStructureMutation){ - adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.toList(), targets) + val name = adaptiveSelectResource(evaluatedIndividual, bySQL = true, candidates.map { it.getFullQualifyingTableName() }, targets) + SqlActionUtils.getTableKey(this.rm.getTableInfo().keys, name) + ?: randomness.choose(candidates) }else{ randomness.choose(candidates) } @@ -196,7 +199,7 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { bySQL: Boolean, candidates: List, targets: Set? - ): TableId{ + ): String{ evaluatedIndividual?: throw IllegalStateException("lack of impact with specified evaluated individual") targets?:throw IllegalStateException("targets must be specified if adaptive resource selection is applied") @@ -204,7 +207,8 @@ class ResourceRestStructureMutator : ApiWsStructureMutator() { throw IllegalStateException("lack of impact info or mismatched impact type (type: ${evaluatedIndividual.impactInfo?.javaClass?.simpleName?:"null"})") val impacts = candidates.map { if (bySQL){ - evaluatedIndividual.impactInfo.sqlTableSizeImpact[it] ?:IntegerGeneImpact("size") + val key = SqlActionUtils.getTableKey(rm.getTableInfo().keys, it) + evaluatedIndividual.impactInfo.sqlTableSizeImpact[key] ?:IntegerGeneImpact("size") }else{ evaluatedIndividual.impactInfo.resourceSizeImpact[it] ?:IntegerGeneImpact("size") } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 665991cf83..d41e7feadf 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -175,7 +175,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { r : RestResourceNode, paramId: String, paramInfo: ParamInfo, - relatedToTables: Set, + relatedToTables: Set, isBodyParam : Boolean, inputIndicator: Int, alltables : Map @@ -269,7 +269,11 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { * @param sqlActions specifies the tables to be analyzed. * if [dbActions] is empty, the tables would be all related tables extracted from its resource node */ - override fun generateRelatedTables(paramsInfo: List, calls: RestResourceCalls, sqlActions : List): MutableMap> { + override fun generateRelatedTables( + paramsInfo: List, + calls: RestResourceCalls, + sqlActions : List + ): MutableMap> { val result = mutableMapOf>() diff --git a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt index 29c0c29f92..dbea7c830b 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/DatabaseExecution.kt @@ -1,7 +1,9 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.execution.SqlExecutionsDto +import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.sql.schema.TableId +import org.slf4j.LoggerFactory /** * When a test case is executed, and the SUT does access a SQL database, @@ -56,7 +58,17 @@ class DatabaseExecution( companion object { - fun fromDto(dto: SqlExecutionsDto?): DatabaseExecution { + private val log = LoggerFactory.getLogger(DatabaseExecution::class.java) + + /** + * From [dto] info on what executed on the SQL database, return an internal representation. + * To achieve this, we need to map it to the existing schema definition from which [tableIds] + * are inferred. + * This is done for few reasons, including: + * 1) collected queries might missing schema info + * 2) there might be cases we don't handle yet (eg views) + */ + fun fromDto(dto: SqlExecutionsDto, tableIds: Set): DatabaseExecution { /* Dealing with quotes in table names is tricky... could be to handle reserved words or case sensitivity. @@ -67,20 +79,44 @@ class DatabaseExecution( might depend on library used to analyze the SQL commands. */ - FIXME - return DatabaseExecution( - cloneData(dealWithQuotes(dto?.queriedData)), - cloneData(dealWithQuotes(dto?.updatedData)), - cloneData(dealWithQuotes(dto?.insertedData)), - cloneData(dealWithQuotes(dto?.failedWhere)), - dealWithQuotes(dto?.deletedData?.toList()) ?: listOf(), - dto?.numberOfSqlCommands ?: 0, - dto?.sqlParseFailureCount ?: 0, - cloneSqlExecutionInfo(dto?.sqlExecutionLogDtoList) + cloneData(convertData(dto.queriedData, tableIds)), + cloneData(convertData(dto.updatedData, tableIds)), + cloneData(convertData(dto.insertedData, tableIds)), + cloneData(convertData(dto.failedWhere, tableIds)), + convertData(dto.deletedData?.toList(), tableIds) ?: listOf(), + dto.numberOfSqlCommands, + dto.sqlParseFailureCount, + cloneSqlExecutionInfo(dto.sqlExecutionLogDtoList) ) } + + private fun convertData(data: List?, tableIds: Set) : List?{ + if(data == null) return null + + return data.mapNotNull { + val id = SqlActionUtils.getTableKey(tableIds, removeQuotes(it)) + if(id == null){ + LoggingUtil.uniqueWarn(log, "Cannot identify table: $it") + } + id + } + } + + private fun convertData(data: Map>?, tableIds: Set) : Map>? { + if(data == null) return null + + return data.entries + .mapNotNull { e -> + val id = SqlActionUtils.getTableKey(tableIds, removeQuotes(e.key)) + if(id == null){ + LoggingUtil.uniqueWarn(log, "Cannot identify table: ${e.key}") + } + id?.let { it to e.value} + }.toMap() + } + fun mergeData( executions: Collection, extractor: (DatabaseExecution) -> Map> @@ -95,21 +131,10 @@ class DatabaseExecution( return data } - private fun dealWithQuotes(data: Map>?): Map>? { - if(data == null) return null - - return data.entries.associate { - (if(it.key.name.startsWith("\"")) removeQuotes(it.key) else it.key) to it.value - } - } - - private fun dealWithQuotes(data: List?) : List?{ - if(data == null) return null - - return data.map { if(it.startsWith("\"")) removeQuotes(it) else it } - } - private fun removeQuotes(s: String) = s.substring(1,s.length-1) + private fun removeQuotes(s: String) = + if(s.startsWith("\"") && s.endsWith("\"")) s.substring(1,s.length-1) + else s private fun cloneData(data: List?): List { if (data == null) { diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 97a60366f3..bc925f1407 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -1049,5 +1049,5 @@ class SqlInsertBuilder( /** * get names of all tables */ - fun getTableNames() = tables.keys + fun getTableNames() : Set = tables.keys } From 0f6c7874177bdca94d379a72ab33ce8fb1b138ab Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 20 Jan 2025 14:40:53 +0100 Subject: [PATCH 21/50] some more refactoring --- .../problem/util/inference/SimpleDeriveR2T.kt | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index d41e7feadf..be9216491e 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -65,7 +65,11 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return@stop } - val matchedPropertyMap = allTables.flatMap { t->t.value.columns.filter { c-> !ParamUtil.isGeneralName(c.name) }.map { c->Pair(t.value.name, c.name) } } + val matchedPropertyMap = allTables + .flatMap { t->t.value.columns + .filter { c-> !ParamUtil.isGeneralName(c.name) } + .map { c->Pair(t.value.id, c.name) } + } .map { p-> Pair(p.first, Pair(p.second, calculateStringSimilarityScoreWithTableName(p.second, token) )) }.asSequence().sortedBy { e->e.second.second } @@ -74,7 +78,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { matchedPropertyMap.filter { it.second.second == matchedPropertyMap.last().second.second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(seg, it.first, similarity = it.second.second, inputIndicator = sindex, outputIndicator = 1)) + }.add(MatchedInfo(seg, it.first.getFullQualifyingTableName(), similarity = it.second.second, inputIndicator = sindex, outputIndicator = 1)) } return@stop } @@ -89,12 +93,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(reftypes.isNotEmpty()){ reftypes.forEach { type-> if(!resourceNode.isPartOfStaticTokens(type) && allTables.isNotEmpty()){ - val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, type)) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), type)) }.asSequence().sortedBy { e->e.second } if(matchedMap.last().second >= StringSimilarityComparator.SimilarityThreshold){ matchedMap.filter { it.second == matchedMap.last().second }.forEach { resourceNode.resourceToTable.derivedMap.getOrPut(it.first){ mutableListOf() - }.add(MatchedInfo(type, it.first, similarity = it.second, inputIndicator = 0, outputIndicator = 0)) + }.add(MatchedInfo(type, it.first.getFullQualifyingTableName(), similarity = it.second, inputIndicator = 0, outputIndicator = 0)) } } } @@ -103,12 +107,12 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { //1.3 derive resource to tables based on tokens on POST action resourceNode.actions.filter {it.verb == HttpVerb.POST }.forEach { post-> post.tokens.values.filter { !resourceNode.getName().toLowerCase().contains(it.getKey().toLowerCase()) }.forEach { atoken-> - val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it, atoken.getKey())) }.asSequence().sortedBy { e->e.second } + val matchedMap = allTables.keys.map { Pair(it, calculateStringSimilarityScoreWithTableName(it.getFullQualifyingTableName(), atoken.getKey())) }.asSequence().sortedBy { e->e.second } matchedMap.last().apply { if(second >= StringSimilarityComparator.SimilarityThreshold){ resourceNode.resourceToTable.derivedMap.getOrPut(first){ mutableListOf() - }.add(MatchedInfo(atoken.getKey(), first, similarity = second, inputIndicator = 1, outputIndicator = 0)) + }.add(MatchedInfo(atoken.getKey(), first.getFullQualifyingTableName(), similarity = second, inputIndicator = 1, outputIndicator = 0)) } } } @@ -216,7 +220,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { */ private fun deriveParamWithTable( paramName : String, - candidateTables : Set, + candidateTables : Set, pToTable : MutableMap, inputlevel: Int, tables : Map){ @@ -227,7 +231,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { private fun deriveParamWithTable( paramName : String, - tableName: String, + tableName: TableId, pToTable : MutableMap, inputlevel: Int, tables : Map @@ -235,7 +239,8 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { /* paramName might be \w+id or \w+name, in this case, we compare paramName with table name of current or referred table + column name */ - getTable(tableName, tables)?.let { t-> + //getTable(tableName, tables)? + tables[tableName]?.let { t-> val matchedMap = t.columns .map { Pair(it.name, From f47182e7384ba8729e373bb60b8b2d9468a28dc3 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Wed, 19 Feb 2025 17:48:19 +0800 Subject: [PATCH 22/50] resolve issues in SimpleDeriveR2T.kt due to updates --- .../core/problem/util/inference/SimpleDeriveR2T.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index be9216491e..669f93a29d 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -187,7 +187,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { if(isBodyParam){ val pToTable = BodyParamRelatedToTable(paramId, paramInfo.referParam) ParamUtil.getObjectGene(paramInfo.referParam.gene)?.fields?.forEach { f-> - val matchedMap : MutableMap = mutableMapOf() + val matchedMap : MutableMap = mutableMapOf() deriveParamWithTable(f.name, relatedToTables, matchedMap, inputIndicator, alltables) if(matchedMap.isNotEmpty()){ val fToTable = ParamFieldRelatedToTable(f.name) @@ -200,7 +200,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return true } }else{ - val matchedMap : MutableMap = mutableMapOf() + val matchedMap : MutableMap = mutableMapOf() deriveParamWithTable(paramInfo.name, relatedToTables, matchedMap, inputIndicator, alltables) if(matchedMap.isNotEmpty()){ val pToTable = SimpleParamRelatedToTable(paramId, paramInfo.referParam) @@ -221,7 +221,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { private fun deriveParamWithTable( paramName : String, candidateTables : Set, - pToTable : MutableMap, + pToTable : MutableMap, inputlevel: Int, tables : Map){ candidateTables.forEach { tableName -> @@ -232,7 +232,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { private fun deriveParamWithTable( paramName : String, tableName: TableId, - pToTable : MutableMap, + pToTable : MutableMap, inputlevel: Int, tables : Map ){ @@ -285,7 +285,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { val missingParams = paramsInfo.map { it.key } val resource = calls.getResourceNode() - val relatedTables = sqlActions.map { it.table.name }.toHashSet() + val relatedTables = sqlActions.map { it.table.id }.toHashSet() val list = if(relatedTables.isEmpty()) getBindMap(missingParams.toSet(), resource.resourceToTable) @@ -325,7 +325,7 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { return result } - private fun getBindMap(paramIds : Set, resourceToTable: ResourceRelatedToTable, tables: Set) : MutableList{ + private fun getBindMap(paramIds : Set, resourceToTable: ResourceRelatedToTable, tables: Set) : MutableList{ val result = mutableListOf() paramIds.forEach { p-> resourceToTable.paramToTable[p]?.let {pToTable-> From 379d758d22746398a1879b389095a7e243ca3ae7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 25 Feb 2025 15:55:31 +0100 Subject: [PATCH 23/50] working on compilation issues --- .../problem/api/service/ApiWsStructureMutator.kt | 2 +- .../core/search/gene/sql/SqlForeignKeyGene.kt | 4 ++++ .../core/search/gene/sql/SqlPrimaryKeyGene.kt | 3 +++ .../core/solver/SMTLibZ3DbConstraintSolver.kt | 6 ++---- .../org/evomaster/core/sql/SqlInsertBuilder.kt | 15 +++++++++++++++ .../test/kotlin/org/evomaster/core/TestUtils.kt | 4 ++-- .../evomaster/core/output/TestCaseWriterTest.kt | 7 ++++--- .../rest/individual/RestIndividualTestBase.kt | 2 +- .../RestResourceIndividualDisabledHMTest.kt | 2 +- 9 files changed, 33 insertions(+), 12 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt index 962298c875..711e926092 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt @@ -397,7 +397,7 @@ abstract class ApiWsStructureMutator : StructureMutator() { return addedSqlInsertions } - private fun handleDSE(sampler: ApiWsSampler, fw: Map>): MutableList> { + private fun handleDSE(ind: T, sampler: ApiWsSampler, failedWhereQueries: List): MutableList> { /* TODO: DSE should be plugged in here */ val schemaDto = sampler.sqlInsertBuilder?.schemaDto ?: throw IllegalStateException("No DB schema is available") diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt index a884f2143e..5ef24660f8 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt @@ -39,6 +39,10 @@ class SqlForeignKeyGene( ) : SqlWrapperGene, SimpleGene(sourceColumn) { + @Deprecated("Rather use the one forcing TableId") + constructor(sourceColumn: String, uniqueId: Long, targetTable: String, nullable: Boolean, uniqueIdOfPrimaryKey: Long =-1) + : this(sourceColumn, uniqueId, TableId(targetTable), nullable, uniqueIdOfPrimaryKey) + init { if (uniqueId < 0) { throw IllegalArgumentException("Negative unique id") diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt index c766d173f1..018801fe75 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlPrimaryKeyGene.kt @@ -29,6 +29,9 @@ class SqlPrimaryKeyGene(name: String, val uniqueId: Long ) : SqlWrapperGene, CompositeGene(name, mutableListOf(gene)) { + @Deprecated("Rather use the one forcing TableId") + constructor(name: String, tableName: String, gene: Gene, uniqueId: Long) : this(name,TableId(tableName),gene, uniqueId) + init { if (uniqueId < 0) { diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt index b63b9b5691..d529d2c9e8 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SMTLibZ3DbConstraintSolver.kt @@ -152,7 +152,7 @@ class SMTLibZ3DbConstraintSolver() : DbConstraintSolver { } val currentColumn = table.columns.firstOrNull(){ it.name == columnName} if (currentColumn != null && currentColumn.primaryKey) { - gene = SqlPrimaryKeyGene(columnName, tableName, gene, idCounter) + gene = SqlPrimaryKeyGene(columnName, table.id, gene, idCounter) idCounter++ } gene.markAllAsInitialized() @@ -203,9 +203,7 @@ class SMTLibZ3DbConstraintSolver() : DbConstraintSolver { ?: throw RuntimeException("Table not found: $tableName") return Table( TableId(tableDto.id.name) , //TODO other info, eg schema - findColumns(tableDto), // Convert columns from DTO - tableDto.name, - findColumns(schema, tableDto), // Convert columns from DTO + findColumns(schema,tableDto), // Convert columns from DTO findForeignKeys(tableDto) // TODO: Implement this method ) } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index dac53db66a..1f1f1253d6 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -658,6 +658,21 @@ class SqlInsertBuilder( return data[tableName] ?: throw IllegalArgumentException("No table called $tableName") } + + @Deprecated("User version with TableId") + fun createSqlInsertionAction( + tableName: String, + columnNames: Set = setOf("*"), + history: MutableList = mutableListOf(), + forceAll: Boolean = false, + useExtraSqlDbConstraints: Boolean = false, + enableSingleInsertionForTable: Boolean = false + ): List { + return createSqlInsertionAction(TableId(tableName), columnNames, history, forceAll,useExtraSqlDbConstraints, enableSingleInsertionForTable) + } + + + /** * Create a SQL insertion operation into the table called [tableName]. * Use columns only from [columnNames], to avoid wasting resources in setting diff --git a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt index a46946016d..82e7831f46 100644 --- a/core/src/test/kotlin/org/evomaster/core/TestUtils.kt +++ b/core/src/test/kotlin/org/evomaster/core/TestUtils.kt @@ -66,7 +66,7 @@ object TestUtils { val fooId = Column("Id", ColumnDataType.INTEGER, 10, primaryKey = true, databaseType = DatabaseType.H2) val foo = Table(TableId(tableName), setOf(fooId), setOf()) val integerGene = IntegerGene(fooId.name, intValue) - val pkFoo = SqlPrimaryKeyGene(fooId.name, "Foo", integerGene, pkGeneUniqueId) + val pkFoo = SqlPrimaryKeyGene(fooId.name, TableId("Foo"), integerGene, pkGeneUniqueId) if(fkColumn != null && fkGene != null) return SqlAction(foo, setOf(fooId, fkColumn), pkId, listOf(pkFoo, fkGene)) return SqlAction(foo, setOf(fooId), pkId, listOf(pkFoo)) @@ -77,7 +77,7 @@ object TestUtils { val fkColumName = "fkId" val fkId = Column(fkColumName, ColumnDataType.INTEGER, 10, primaryKey = false, databaseType = DatabaseType.H2) - val foreignKeyGene = SqlForeignKeyGene(fkColumName, bId, aTable, false, uniqueIdOfPrimaryKey = aUniqueId) + val foreignKeyGene = SqlForeignKeyGene(fkColumName, bId, TableId(aTable), false, uniqueIdOfPrimaryKey = aUniqueId) val barInsertion = generateFakeDbAction(bId, bUniqueId, bTable, bValue, fkId, foreignKeyGene) diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index 1f4b75c8f8..b7edb77401 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -25,6 +25,7 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.gene.UUIDGene import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.string.StringGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import javax.ws.rs.core.MediaType @@ -269,7 +270,7 @@ class TestCaseWriterTest : WriterTestBase(){ val id = 0L val integerGene = IntegerGene(idColumn.name, 42, 0, 100) - val primaryKeyGene = SqlPrimaryKeyGene(idColumn.name, "myTable", integerGene, 10) + val primaryKeyGene = SqlPrimaryKeyGene(idColumn.name, TableId("myTable"), integerGene, 10) val stringGene = StringGene(nameColumn.name, "nameValue", 0, 10) val insertIntoTableAction = SqlAction(aTable, setOf(idColumn, nameColumn), id, listOf(primaryKeyGene, stringGene)) @@ -316,8 +317,8 @@ class TestCaseWriterTest : WriterTestBase(){ val pkGeneUniqueId = 12345L val integerGene = IntegerGene(idColumn.name, 42, 0, 100) - val primaryKeyTable0Gene = SqlPrimaryKeyGene(idColumn.name, "Table0", integerGene, pkGeneUniqueId) - val primaryKeyTable1Gene = SqlPrimaryKeyGene(idColumn.name, "Table1", integerGene, 10) + val primaryKeyTable0Gene = SqlPrimaryKeyGene(idColumn.name, TableId("Table0"), integerGene, pkGeneUniqueId) + val primaryKeyTable1Gene = SqlPrimaryKeyGene(idColumn.name, TableId("Table1"), integerGene, 10) val firstInsertionId = 1001L diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt index c35e71cf1a..2612eb1dff 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt @@ -611,7 +611,7 @@ abstract class RestIndividualTestBase { .apply { val table = randomness.choose( sqlInsertBuilder!!.getTableNames()) val failed = randomness.choose(sqlInsertBuilder!!.getTable(table,true).columns.map { it.name }) - failedWhere = mapOf(table to setOf(failed)) + failedWhere = mapOf(table.name to setOf(failed)) } } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt index 0227a17ca7..dead157fcc 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt @@ -83,7 +83,7 @@ class RestResourceIndividualDisabledHMTest : RestIndividualTestBase(){ override fun getFitnessFunction(): AbstractRestFitness = ff private fun sampleDbAction(table : Table) : List{ - val actions = sqlInsertBuilder!!.createSqlInsertionAction(table.name) + val actions = sqlInsertBuilder!!.createSqlInsertionAction(table) return actions.map { it.copy() as SqlAction } } From d8aac639a7455d53be5a9aae58ab9aab8b7d3451 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 25 Feb 2025 21:33:50 +0100 Subject: [PATCH 24/50] fixed some more compilation issues --- .../RestResourceIndividualDisabledHMTest.kt | 2 +- .../rest/resource/ResourceNodeWithDbTest.kt | 43 ++++++++++--------- .../postgres/ManySimilarToChecksTest.kt | 3 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt index dead157fcc..b83bc0ff5f 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestResourceIndividualDisabledHMTest.kt @@ -83,7 +83,7 @@ class RestResourceIndividualDisabledHMTest : RestIndividualTestBase(){ override fun getFitnessFunction(): AbstractRestFitness = ff private fun sampleDbAction(table : Table) : List{ - val actions = sqlInsertBuilder!!.createSqlInsertionAction(table) + val actions = sqlInsertBuilder!!.createSqlInsertionAction(table.id) return actions.map { it.copy() as SqlAction } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index cb1b084abd..1a60af20c5 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -15,6 +15,7 @@ import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.service.Randomness import org.evomaster.core.sql.SqlActionUtils.isMatchingTableName +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -70,17 +71,17 @@ class ResourceNodeWithDbTest { assertEquals(6, cluster.getCluster().size) // table in db - assertTrue(cluster.getTableInfo().keys.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ"))) + assertTrue(cluster.getTableInfo().keys.map { it.getFullQualifyingTableName() }.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ"))) // data in db - assertEquals(2, cluster.getDataInDb("RFOO")?.size) - assertEquals(1, cluster.getDataInDb("RBAR")?.size) - assertEquals(1, cluster.getDataInDb("RXYZ")?.size) + assertEquals(2, cluster.getDataInDb(TableId("RFOO"))?.size) + assertEquals(1, cluster.getDataInDb(TableId("RBAR"))?.size) + assertEquals(1, cluster.getDataInDb(TableId("RXYZ"))?.size) val rfooNode = cluster.getResourceNode("/v3/api/rfoo") assertNotNull(rfooNode) rfooNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.contains("PUBLIC.RFOO"), "Keys: ${derivedMap.keys.joinToString(", ")}") + assertTrue(derivedMap.keys.map { it.getFullQualifyingTableName() }.contains("PUBLIC.RFOO"), "Keys: ${derivedMap.keys.joinToString(", ")}") assertEquals(1, paramToTable.size) assertTrue(paramToTable.values.first() is BodyParamRelatedToTable) (paramToTable.values.first() as BodyParamRelatedToTable).apply { @@ -89,7 +90,7 @@ class ResourceNodeWithDbTest { assertEquals(1, u.derivedMap.size) u.derivedMap.forEach { ut, uu -> // assertEquals("RFOO", ut) - assertTrue(isMatchingTableName(ut, "RFOO")) + assertTrue(isMatchingTableName(ut.name, "RFOO")) assertEquals(uu.input.toLowerCase(), uu.targetMatched.toLowerCase()) } } @@ -99,38 +100,38 @@ class ResourceNodeWithDbTest { val rbarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}") assertNotNull(rbarNode) rbarNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) - assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it.name, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it.name, "RBAR") }) } val rxyzNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}") assertNotNull(rxyzNode) rxyzNode!!.resourceToTable.apply { - assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RFOO") }) - assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RBAR") }) - assertTrue(derivedMap.keys.any { isMatchingTableName(it, "RXYZ") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it.name, "RFOO") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it.name, "RBAR") }) + assertTrue(derivedMap.keys.any { isMatchingTableName(it.name, "RXYZ") }) } } @Test fun testDbActionCreation(){ - val fooAndBar = cluster.createSqlAction(listOf("RFOO","RBAR"), sqlInsertBuilder, mutableListOf(), true, randomness= randomness) + val fooAndBar = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) assertEquals(2, fooAndBar.size) - val fooAndBar2 = cluster.createSqlAction(listOf("RFOO","RBAR"), sqlInsertBuilder, fooAndBar, true, randomness= randomness) + val fooAndBar2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, fooAndBar, true, randomness= randomness) assertEquals(0, fooAndBar2.size) - val xyz = cluster.createSqlAction(listOf("RXYZ"), sqlInsertBuilder, mutableListOf(), true, randomness= randomness) + val xyz = cluster.createSqlAction(listOf("RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) assertEquals(3, xyz.size) - val xyz2 = cluster.createSqlAction(listOf("RFOO","RBAR"), sqlInsertBuilder, xyz, true, randomness= randomness) + val xyz2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, xyz, true, randomness= randomness) assertEquals(0, xyz2.size) - val xyz3 = cluster.createSqlAction(listOf("RFOO","RFOO","RBAR","RBAR"), sqlInsertBuilder, mutableListOf(), false, randomness= randomness) + val xyz3 = cluster.createSqlAction(listOf("RFOO","RFOO","RBAR","RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), false, randomness= randomness) assertEquals(2 + 2*2, xyz3.size) - val xyzSelect = cluster.createSqlAction(listOf("RXYZ"), sqlInsertBuilder, mutableListOf(), true, isInsertion = false, randomness = randomness) + val xyzSelect = cluster.createSqlAction(listOf("RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, isInsertion = false, randomness = randomness) assertEquals(1, xyzSelect.size) } @@ -171,7 +172,7 @@ class ResourceNodeWithDbTest { // /v3/api/rfoo/{rfooId}/rbar/{rbarId} val getBarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}")!! val getBar = getBarNode.sampleRestResourceCalls("GET", randomness, maxTestSize = 10) - val fooBarDbActionToCreate = cluster.createSqlAction(listOf("RFOO", "RBAR"), sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val fooBarDbActionToCreate = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) assertEquals(2, fooBarDbActionToCreate.size) getBar.initDbActions(fooBarDbActionToCreate, cluster, false, false) val barFooId = getGenePredict(getBar.seeActions(ActionFilter.NO_SQL).first(), "rfooId"){ g: Gene-> g is LongGene } @@ -186,7 +187,7 @@ class ResourceNodeWithDbTest { // /v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId} val xYZNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}")!! val getXYZ = xYZNode.sampleRestResourceCalls("GET", randomness, 10) - val xyzDbActions = cluster.createSqlAction(listOf("RXYZ", "RBAR", "RFOO"), sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val xyzDbActions = cluster.createSqlAction(listOf("RXYZ", "RBAR", "RFOO").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getGenePredict(xyzDbActions[0], "id"){g: Gene-> g is LongGene }.apply { (this as? LongGene)?.value = 42 } @@ -221,7 +222,7 @@ class ResourceNodeWithDbTest { val xYZNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}")!! val getXYZ = xYZNode.sampleRestResourceCalls("GET", randomness, 10) - val dbXYZ = cluster.createSqlAction(listOf("RFOO", "RBAR", "RXYZ"), sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val dbXYZ = cluster.createSqlAction(listOf("RFOO", "RBAR", "RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getGenePredict(dbXYZ[0], "id"){g: Gene-> g is LongGene }.apply { (this as? LongGene)?.value = 42 } @@ -241,7 +242,7 @@ class ResourceNodeWithDbTest { val getBarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}")!! val getBar = getBarNode.sampleRestResourceCalls("GET", randomness, maxTestSize = 10) - val dbBar = cluster.createSqlAction(listOf("RFOO", "RBAR"), sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val dbBar = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getBar.initDbActions(dbBar, cluster, false, false) assertEquals(2, getBar.seeActionSize(ActionFilter.ONLY_SQL)) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt index f91b9bb1d0..7c4b8ec234 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt @@ -6,6 +6,7 @@ import org.evomaster.core.sql.SqlActionTransformer import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.regex.RegexGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Disabled @@ -27,7 +28,7 @@ class ManySimilarToChecksTest : ExtractTestBasePostgres() { assertEquals(2, schema.tables.first { it.id.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) val builder = SqlInsertBuilder(schema) - val emailTable = builder.getTable("email_table", useExtraConstraints = true) + val emailTable = builder.getTable(TableId("email_table"), useExtraConstraints = true) assertEquals(1, emailTable.columns.size) val emailColumn = emailTable.columns.first() From afcd01e2f9149d3c973701b013ef55290e36f18a Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 6 Mar 2025 13:43:45 +0100 Subject: [PATCH 25/50] fixed series of broken tests --- .../java/sql/heuristic/SqlExpressionEvaluator.java | 2 +- .../core/problem/rest/service/ResourceRestModule.kt | 9 +++++++-- .../org/evomaster/core/sql/SqlInsertBuilder.kt | 12 +++++++++--- .../kotlin/org/evomaster/core/sql/schema/TableId.kt | 5 +++++ .../org/evomaster/core/sql/SqlInsertBuilderTest.kt | 6 +++--- .../core/sql/extract/h2/CatwatchSqlExtractTest.kt | 4 ++-- 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java index 3ab417cdb4..01629e47cf 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java @@ -619,7 +619,7 @@ public void visit(ParenthesedSelect parenthesedSelect) { @Override public void visit(Column column) { String name = column.getColumnName(); - String table = sqlNameContext.getTableName(column); + String table = sqlNameContext.getFullyQualifiedTableName(column); Object value = dataRow.getValueByName(name, table); concreteValues.push(value); } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestModule.kt index 805148a9cc..8781ca0c06 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceRestModule.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.rest.service import com.google.inject.TypeLiteral +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler import org.evomaster.core.problem.rest.RestIndividual @@ -29,9 +30,13 @@ class ResourceRestModule(private val bindRemote : Boolean = true) : RestBaseModu .asEagerSingleton() } + + bind(object : TypeLiteral>() {}) + .to(ResourceSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) - .to(ResourceSampler::class.java) - .asEagerSingleton() + .to(ResourceSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral() {}) .to(ResourceSampler::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt index 1f1f1253d6..5fef1b3d08 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt @@ -655,11 +655,14 @@ class SqlInsertBuilder( // val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName) // ?: throw IllegalArgumentException("No table called $tableName") - return data[tableName] ?: throw IllegalArgumentException("No table called $tableName") + val key = data.keys.find { tableName.isEquivalentIgnoringCase(it) } + ?: throw IllegalArgumentException("No table called ${tableName.getFullQualifyingTableName()}") + + return data[key]!! } - @Deprecated("User version with TableId") + @Deprecated("Use version with TableId") fun createSqlInsertionAction( tableName: String, columnNames: Set = setOf("*"), @@ -668,7 +671,10 @@ class SqlInsertBuilder( useExtraSqlDbConstraints: Boolean = false, enableSingleInsertionForTable: Boolean = false ): List { - return createSqlInsertionAction(TableId(tableName), columnNames, history, forceAll,useExtraSqlDbConstraints, enableSingleInsertionForTable) + + val id = TableId(tableName.lowercase(), openGroupName = "public") + + return createSqlInsertionAction(id, columnNames, history, forceAll,useExtraSqlDbConstraints, enableSingleInsertionForTable) } diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt index 865b750f76..471f675cee 100644 --- a/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt +++ b/core/src/main/kotlin/org/evomaster/core/sql/schema/TableId.kt @@ -80,4 +80,9 @@ data class TableId( } return "$openGroupName.${name}" } + + fun isEquivalentIgnoringCase(other: TableId) : Boolean{ + return getFullQualifyingTableName().equals(other.getFullQualifyingTableName(), true) + && connectionId == other.connectionId + } } diff --git a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt index 28ab5568a3..9a07c6eeef 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/SqlInsertBuilderTest.kt @@ -1067,9 +1067,9 @@ class SqlInsertBuilderTest { // ABCD or ACBD assertEquals(4, enabled.size) - assertEquals("PUBLIC.ROOTA", enabled[0].table.name) - assertEquals("PUBLIC.LEAFD", enabled[3].table.name) - assertTrue(enabled.subList(1,3).map { it.table.name }.containsAll(listOf("PUBLIC.NODEB", "PUBLIC.NODEC"))) + assertEquals("PUBLIC.ROOTA", enabled[0].table.id.getFullQualifyingTableName().uppercase()) + assertEquals("PUBLIC.LEAFD", enabled[3].table.id.getFullQualifyingTableName().uppercase()) + assertTrue(enabled.subList(1,3).map { it.table.id.getFullQualifyingTableName().uppercase() }.containsAll(listOf("PUBLIC.NODEB", "PUBLIC.NODEC"))) } ////// Tests Numeric parsing for Min/Max Bounds diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt index f352d964eb..e5753307dc 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/h2/CatwatchSqlExtractTest.kt @@ -59,8 +59,8 @@ class CatwatchSqlExtractTest : ExtractTestBaseH2(){ SqlActionUtils.randomizeDbActionGenes(insertions.toMutableList(), Randomness()) assertEquals(2, insertions.size) - assert(insertions[0].table.name.equals("PUBLIC.PROJECT", ignoreCase = true)) - assert(insertions[1].table.name.equals("PUBLIC.lANGUAGE_LIST", ignoreCase = true)) + assert(insertions[0].table.id.getFullQualifyingTableName().equals("PUBLIC.PROJECT", ignoreCase = true)) + assert(insertions[1].table.id.getFullQualifyingTableName().equals("PUBLIC.lANGUAGE_LIST", ignoreCase = true)) val projectId = (insertions[0].seeTopGenes().filterIsInstance()).first().uniqueId From fad99829d800600e3a15ae03d13c7f9a59d75318 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 6 Mar 2025 14:29:28 +0100 Subject: [PATCH 26/50] fixed more tests --- .../extract/mysql/CreateTableBoundedNumberTest.kt | 5 ++++- .../core/sql/extract/mysql/CreateTableEnumTest.kt | 3 ++- .../core/sql/extract/mysql/ExtractTestBaseMySQL.kt | 2 +- .../core/sql/extract/mysql/LikeCheckTest.kt | 5 +++-- .../core/sql/extract/mysql/SQLJSONColumnTest.kt | 13 +++++++------ .../core/sql/extract/mysql/SqlDateColumnTest.kt | 5 +++-- .../sql/extract/mysql/SqlDateTypesColumnTest.kt | 3 ++- .../core/sql/extract/mysql/SqlTextColumnTest.kt | 7 ++++--- .../sql/extract/postgres/ManySimilarToChecksTest.kt | 2 +- 9 files changed, 27 insertions(+), 18 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt index 165b124d3b..7a4b446e0c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt @@ -8,6 +8,7 @@ import org.evomaster.core.search.gene.numeric.DoubleGene import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.gene.optional.NullableGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import java.math.BigDecimal @@ -47,7 +48,9 @@ class CreateTableBoundedNumberTest : ExtractTestBaseMySQL() { val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("BoundedNumberTable", setOf("*")) + val actions = builder.createSqlInsertionAction( + TableId("BoundedNumberTable", openGroupName = MYSQL_DB_NAME) + , setOf("*")) actions[0].seeTopGenes().apply { assertEquals(8, size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt index e4cf633c1f..711d4e4ad1 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt @@ -4,6 +4,7 @@ import org.evomaster.client.java.sql.DbInfoExtractor import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.collection.EnumGene import org.evomaster.core.search.gene.optional.NullableGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -32,7 +33,7 @@ class CreateTableEnumTest : ExtractTestBaseMySQL() { assertEquals("size enum('x-small','small','medium','large','x-large')",tableDto.tableCheckExpressions[0].sqlCheckExpression.toLowerCase()) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("shirts", setOf("name", "size")) + val actions = builder.createSqlInsertionAction(TableId("shirts", openGroupName = MYSQL_DB_NAME), setOf("name", "size")) val sizeGene = actions[0].seeTopGenes().find { it.name == "size" } assertTrue(sizeGene is NullableGene) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/ExtractTestBaseMySQL.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/ExtractTestBaseMySQL.kt index c9b7a92876..fe0ff6533b 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/ExtractTestBaseMySQL.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/ExtractTestBaseMySQL.kt @@ -20,7 +20,7 @@ abstract class ExtractTestBaseMySQL { private var sqlSchemaCommand : String? = null - private const val MYSQL_DB_NAME = "test" + public const val MYSQL_DB_NAME = "test" private const val MYSQL_PORT = 3306 diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/LikeCheckTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/LikeCheckTest.kt index 44c982fac6..e709d1c715 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/LikeCheckTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/LikeCheckTest.kt @@ -6,6 +6,7 @@ import org.evomaster.core.sql.SqlActionTransformer import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.regex.RegexGene import org.evomaster.core.search.service.Randomness +import org.evomaster.core.sql.schema.TableId import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -19,7 +20,7 @@ class LikeCheckTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("x", setOf("f_id")) + val actions = builder.createSqlInsertionAction(TableId("x", openGroupName = MYSQL_DB_NAME), setOf("f_id")) val genes = actions[0].seeTopGenes() assertEquals(1, genes.size) @@ -32,7 +33,7 @@ class LikeCheckTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("x", setOf("f_id")) + val actions = builder.createSqlInsertionAction(TableId("x", openGroupName = MYSQL_DB_NAME), setOf("f_id")) val genes = actions[0].seeTopGenes() val regexGene = genes.firstIsInstance() diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SQLJSONColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SQLJSONColumnTest.kt index 64326a0566..4c401eda42 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SQLJSONColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SQLJSONColumnTest.kt @@ -12,6 +12,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.sql.SqlJSONGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.gene.string.StringGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -25,7 +26,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val genes = actions[0].seeTopGenes() assertEquals(2, genes.size) @@ -39,7 +40,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val genes = actions[0].seeTopGenes() val idValue = ((genes[0] as SqlPrimaryKeyGene).gene as IntegerGene).value @@ -69,7 +70,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val tableDto = schema.tables[0] val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val action = actions[0] val genes = action.seeTopGenes() @@ -113,7 +114,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val tableDto = schema.tables[0] val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val action = actions[0] val genes = action.seeTopGenes() @@ -154,7 +155,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val tableDto = schema.tables[0] val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val action = actions[0] val genes = action.seeTopGenes() @@ -201,7 +202,7 @@ class SQLJSONColumnTest : ExtractTestBaseMySQL() { val tableDto = schema.tables[0] val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "jsonData")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "jsonData")) val action = actions[0] val genes = action.seeTopGenes() diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateColumnTest.kt index 562f598785..4db122e9ba 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateColumnTest.kt @@ -7,6 +7,7 @@ import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.datetime.DateGene import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import java.sql.Date @@ -20,7 +21,7 @@ class SqlDateColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("logins", setOf("userId", "lastLogin")) + val actions = builder.createSqlInsertionAction(TableId("logins", openGroupName = MYSQL_DB_NAME), setOf("userId", "lastLogin")) val genes = actions[0].seeTopGenes() assertEquals(2, genes.size) @@ -34,7 +35,7 @@ class SqlDateColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("logins", setOf("userId", "lastLogin")) + val actions = builder.createSqlInsertionAction(TableId("logins", openGroupName = MYSQL_DB_NAME), setOf("userId", "lastLogin")) val genes = actions[0].seeTopGenes() val userIdValue = ((genes[0] as SqlPrimaryKeyGene).gene as IntegerGene).value val lastLoginDayValue = (genes[1] as DateGene).day.value diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateTypesColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateTypesColumnTest.kt index 49d1efe08e..d0a6f9b014 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateTypesColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlDateTypesColumnTest.kt @@ -4,6 +4,7 @@ import org.evomaster.client.java.sql.DbInfoExtractor import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.datetime.DateTimeGene import org.evomaster.core.search.gene.datetime.TimeGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -16,7 +17,7 @@ class SqlDateTypesColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("t1", setOf("t", "dt", "ts")) + val actions = builder.createSqlInsertionAction(TableId("t1", openGroupName = MYSQL_DB_NAME), setOf("t", "dt", "ts")) val genes = actions[0].seeTopGenes() assertEquals(3, genes.size) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlTextColumnTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlTextColumnTest.kt index 79836c2b70..e7a1109ee1 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlTextColumnTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/SqlTextColumnTest.kt @@ -8,6 +8,7 @@ import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.optional.NullableGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene +import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -21,7 +22,7 @@ class SqlTextColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "name", "address")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "name", "address")) val genes = actions[0].seeTopGenes() assertEquals(3, genes.size) @@ -35,7 +36,7 @@ class SqlTextColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "name", "address")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "name", "address")) val genes = actions[0].seeTopGenes() val idValue = ((genes[0] as SqlPrimaryKeyGene).gene as IntegerGene).value @@ -64,7 +65,7 @@ class SqlTextColumnTest : ExtractTestBaseMySQL() { val schema = DbInfoExtractor.extract(connection) val builder = SqlInsertBuilder(schema) - val actions = builder.createSqlInsertionAction("people", setOf("id", "name", "address")) + val actions = builder.createSqlInsertionAction(TableId("people", openGroupName = MYSQL_DB_NAME), setOf("id", "name", "address")) val genes = actions[0].seeTopGenes() val idValue = ((genes[0] as SqlPrimaryKeyGene).gene as IntegerGene).value diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt index 7c4b8ec234..9b9a5b60c4 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/postgres/ManySimilarToChecksTest.kt @@ -28,7 +28,7 @@ class ManySimilarToChecksTest : ExtractTestBasePostgres() { assertEquals(2, schema.tables.first { it.id.name.equals("email_table", ignoreCase = true) }.tableCheckExpressions.size) val builder = SqlInsertBuilder(schema) - val emailTable = builder.getTable(TableId("email_table"), useExtraConstraints = true) + val emailTable = builder.getTable(TableId("email_table", openGroupName = "public"), useExtraConstraints = true) assertEquals(1, emailTable.columns.size) val emailColumn = emailTable.columns.first() From 504b647bc365b82ab376d9adbe2f11086018ec58 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 6 Mar 2025 14:52:15 +0100 Subject: [PATCH 27/50] more fixes --- .../problem/rest/resource/ResourceCluster.kt | 5 +++-- .../rest/resource/ResourceNodeWithDbTest.kt | 22 +++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt index 953a0fafff..ed16e17256 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt @@ -130,7 +130,7 @@ class ResourceCluster { fun getDataInDb(tableName: TableId) : MutableList?{ // val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName) // ?: return null - return dataInDB.getValue(tableName) + return dataInDB.entries.find { it.key.isEquivalentIgnoringCase(tableName) }?.value } /** @@ -166,7 +166,8 @@ class ResourceCluster { enableSingleInsertionForTable : Boolean = false ) : MutableList{ val sorted = SqlActionUtils.sortTable( - tables.mapNotNull { this.tables[it] }.run { if (doNotCreateDuplicatedAction) this.distinct() else this } + tables.mapNotNull { t -> this.tables.entries.find { it.key.isEquivalentIgnoringCase(t) }?.value } + .run { if (doNotCreateDuplicatedAction) this.distinct() else this } ) val added = mutableListOf() val preTables = previous.filter { !isInsertion || !it.representExistingData }.map { it.table.name }.toMutableList() diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index 1a60af20c5..4e8f34cb60 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -74,9 +74,9 @@ class ResourceNodeWithDbTest { assertTrue(cluster.getTableInfo().keys.map { it.getFullQualifyingTableName() }.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ"))) // data in db - assertEquals(2, cluster.getDataInDb(TableId("RFOO"))?.size) - assertEquals(1, cluster.getDataInDb(TableId("RBAR"))?.size) - assertEquals(1, cluster.getDataInDb(TableId("RXYZ"))?.size) + assertEquals(2, cluster.getDataInDb(TableId("RFOO", openGroupName = "public"))?.size) + assertEquals(1, cluster.getDataInDb(TableId("RBAR", openGroupName = "public"))?.size) + assertEquals(1, cluster.getDataInDb(TableId("RXYZ", openGroupName = "public"))?.size) val rfooNode = cluster.getResourceNode("/v3/api/rfoo") assertNotNull(rfooNode) @@ -116,22 +116,22 @@ class ResourceNodeWithDbTest { @Test fun testDbActionCreation(){ - val fooAndBar = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) + val fooAndBar = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) assertEquals(2, fooAndBar.size) - val fooAndBar2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, fooAndBar, true, randomness= randomness) + val fooAndBar2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, fooAndBar, true, randomness= randomness) assertEquals(0, fooAndBar2.size) - val xyz = cluster.createSqlAction(listOf("RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) + val xyz = cluster.createSqlAction(listOf("RXYZ").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), true, randomness= randomness) assertEquals(3, xyz.size) - val xyz2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it) }, sqlInsertBuilder, xyz, true, randomness= randomness) + val xyz2 = cluster.createSqlAction(listOf("RFOO","RBAR").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, xyz, true, randomness= randomness) assertEquals(0, xyz2.size) - val xyz3 = cluster.createSqlAction(listOf("RFOO","RFOO","RBAR","RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), false, randomness= randomness) + val xyz3 = cluster.createSqlAction(listOf("RFOO","RFOO","RBAR","RBAR").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), false, randomness= randomness) assertEquals(2 + 2*2, xyz3.size) - val xyzSelect = cluster.createSqlAction(listOf("RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, isInsertion = false, randomness = randomness) + val xyzSelect = cluster.createSqlAction(listOf("RXYZ").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), true, isInsertion = false, randomness = randomness) assertEquals(1, xyzSelect.size) } @@ -222,7 +222,7 @@ class ResourceNodeWithDbTest { val xYZNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}")!! val getXYZ = xYZNode.sampleRestResourceCalls("GET", randomness, 10) - val dbXYZ = cluster.createSqlAction(listOf("RFOO", "RBAR", "RXYZ").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val dbXYZ = cluster.createSqlAction(listOf("RFOO", "RBAR", "RXYZ").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getGenePredict(dbXYZ[0], "id"){g: Gene-> g is LongGene }.apply { (this as? LongGene)?.value = 42 } @@ -242,7 +242,7 @@ class ResourceNodeWithDbTest { val getBarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}")!! val getBar = getBarNode.sampleRestResourceCalls("GET", randomness, maxTestSize = 10) - val dbBar = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val dbBar = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it, openGroupName = "public") }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getBar.initDbActions(dbBar, cluster, false, false) assertEquals(2, getBar.seeActionSize(ActionFilter.ONLY_SQL)) From 7b29cbb6346be46bbd9087686e84f4d115c747b9 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 6 Mar 2025 14:59:15 +0100 Subject: [PATCH 28/50] yet again more fixes --- .../org/evomaster/core/problem/rest/service/RestModule.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestModule.kt index a68bbccbc4..d76530ef1d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestModule.kt @@ -1,6 +1,7 @@ package org.evomaster.core.problem.rest.service import com.google.inject.TypeLiteral +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler import org.evomaster.core.problem.rest.RestIndividual @@ -25,8 +26,11 @@ class RestModule(private val bindRemote : Boolean = true) : RestBaseModule(){ } bind(object : TypeLiteral>() {}) - .to(RestSampler::class.java) - .asEagerSingleton() + .to(RestSampler::class.java) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) + .to(RestSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(RestSampler::class.java) From a5f9ffe84da009d55c4a999ee7a1bf0e949c237d Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Sun, 9 Mar 2025 22:09:07 +0800 Subject: [PATCH 29/50] fix --- .../core/problem/util/inference/SimpleDeriveR2T.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt index 669f93a29d..fce94f0607 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/util/inference/SimpleDeriveR2T.kt @@ -245,14 +245,14 @@ object SimpleDeriveResourceBinding : DeriveResourceBinding { .map { Pair(it.name, if(ParamUtil.isGeneralName(it.name)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTableId}${it.name}")} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName(paramName, "${f.targetTableId.name}${it.name}")} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) - .plus(calculateStringSimilarityScoreWithTableName(paramName, "$tableName${it.name}")) + .plus(calculateStringSimilarityScoreWithTableName(paramName, "${tableName.name}${it.name}")) .asSequence().sorted().last() else if(ParamUtil.isGeneralName(paramName)) - t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTableId}$paramName", it.name)} + t.foreignKeys.map { f-> calculateStringSimilarityScoreWithTableName("${f.targetTableId.name}$paramName", it.name)} .plus(calculateStringSimilarityScoreWithTableName(paramName, it.name)) - .plus(calculateStringSimilarityScoreWithTableName("$tableName$paramName", it.name)) + .plus(calculateStringSimilarityScoreWithTableName("${tableName.name}$paramName", it.name)) .asSequence().sorted().last() else calculateStringSimilarityScoreWithTableName(paramName, it.name)) }.asSequence().sortedBy { e->e.second } From 5e2e0a9cb9c05157d6302fe2097fa24877ec160a Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Sun, 9 Mar 2025 22:09:46 +0800 Subject: [PATCH 30/50] fix test --- .../core/problem/rest/resource/ResourceNodeWithDbTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt index 4e8f34cb60..f5c9e21a39 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt @@ -172,7 +172,7 @@ class ResourceNodeWithDbTest { // /v3/api/rfoo/{rfooId}/rbar/{rbarId} val getBarNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}")!! val getBar = getBarNode.sampleRestResourceCalls("GET", randomness, maxTestSize = 10) - val fooBarDbActionToCreate = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val fooBarDbActionToCreate = cluster.createSqlAction(listOf("RFOO", "RBAR").map { TableId(it, sealedGroupName = "DB_TEST", openGroupName = "PUBLIC") }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) assertEquals(2, fooBarDbActionToCreate.size) getBar.initDbActions(fooBarDbActionToCreate, cluster, false, false) val barFooId = getGenePredict(getBar.seeActions(ActionFilter.NO_SQL).first(), "rfooId"){ g: Gene-> g is LongGene } @@ -187,7 +187,7 @@ class ResourceNodeWithDbTest { // /v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId} val xYZNode = cluster.getResourceNode("/v3/api/rfoo/{rfooId}/rbar/{rbarId}/rxyz/{rxyzId}")!! val getXYZ = xYZNode.sampleRestResourceCalls("GET", randomness, 10) - val xyzDbActions = cluster.createSqlAction(listOf("RXYZ", "RBAR", "RFOO").map { TableId(it) }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) + val xyzDbActions = cluster.createSqlAction(listOf("RXYZ", "RBAR", "RFOO").map { TableId(it, sealedGroupName = "DB_TEST", openGroupName = "PUBLIC") }, sqlInsertBuilder, mutableListOf(), true, randomness = randomness) getGenePredict(xyzDbActions[0], "id"){g: Gene-> g is LongGene }.apply { (this as? LongGene)?.value = 42 } From 1a013a3403bec652a79633cd0604561fb854e852 Mon Sep 17 00:00:00 2001 From: Man Zhang Date: Sun, 9 Mar 2025 22:21:55 +0800 Subject: [PATCH 31/50] fix compilation error --- .../rest/service/resource/ResourceTestBase.kt | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/service/resource/ResourceTestBase.kt b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/service/resource/ResourceTestBase.kt index b222b31fad..80cfec5f20 100644 --- a/core-it/src/test/kotlin/org/evomaster/core/problem/rest/service/resource/ResourceTestBase.kt +++ b/core-it/src/test/kotlin/org/evomaster/core/problem/rest/service/resource/ResourceTestBase.kt @@ -192,11 +192,15 @@ abstract class ResourceTestBase : ExtractTestBaseH2(), ResourceBasedTestInterfac fun testResourceRelatedToTable(resource: String, expectedRelatedTable : List){ assertTrue(rm.getTableInfo().isNotEmpty()) - rm.getResourceCluster().getValue(resource).getDerivedTables().apply { + rm.getResourceCluster().getValue(resource).getDerivedTables(rm.sqlInsertBuilder!!.getTableNames()).apply { assertTrue(isNotEmpty()){ "derived tables of resource $resource should not be empty" } - assertTrue(expectedRelatedTable.all { this.any { a->a.equals(it, ignoreCase = true) } }){ + /* + here, we employ with the table + TODO might later change it with table id + */ + assertTrue(expectedRelatedTable.all { this.any { a->a.name.equals(it, ignoreCase = true) } }){ "expected related tables are ${expectedRelatedTable.joinToString(",")}; but actual related tables are ${this.joinToString(",")}." } } @@ -230,7 +234,11 @@ abstract class ResourceTestBase : ExtractTestBaseH2(), ResourceBasedTestInterfac assertTrue(resourceNode.resourceToTable.derivedMap.isNotEmpty()) - assertTrue(resourceNode.resourceToTable.derivedMap.any { tables.any { t-> it.key.equals(t.key, ignoreCase = true) } }) + /* + here, we employ with the table + TODO might later change it with table id + */ + assertTrue(resourceNode.resourceToTable.derivedMap.any { tables.any { t-> it.key.name.equals(t.key, ignoreCase = true) } }) val resourceCalls = mutableListOf() rm.sampleCall(resourceNode.getName(), true, resourceCalls, config.maxTestSize, true) From ed7d74ddd4dbc24ed29aefa57087fbcd055bf0d7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 11 Mar 2025 10:30:26 +0100 Subject: [PATCH 32/50] fixed NPE --- .../core/problem/enterprise/service/EnterpriseFitness.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index b7e5e1b10c..c5828431a6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -258,7 +258,7 @@ abstract class EnterpriseFitness : FitnessFunction() where T : Individual for (i in 0 until dto.extraHeuristics.size) { val extra = dto.extraHeuristics[i] val sdto = extra.sqlSqlExecutionsDto - if(sdto != null) { + if(sampler.sqlInsertBuilder != null && sdto != null) { val databaseExecution = DatabaseExecution.fromDto(sdto, sampler.sqlInsertBuilder!!.getTableNames()) fv.setDatabaseExecution(i, databaseExecution) if (databaseExecution.sqlParseFailureCount > 0) { From 6054ef4c58b827ce61d4751c2a4947ad84da37e9 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 11 Mar 2025 12:45:35 +0100 Subject: [PATCH 33/50] fixed DI --- .../evomaster/core/problem/rest/service/BlackBoxRestModule.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/BlackBoxRestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/BlackBoxRestModule.kt index c99fb7a0aa..f1c36c22d3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/BlackBoxRestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/BlackBoxRestModule.kt @@ -5,6 +5,7 @@ import com.google.inject.TypeLiteral import org.evomaster.core.output.service.RestTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation @@ -22,6 +23,9 @@ class BlackBoxRestModule( super.configure() + bind(object : TypeLiteral>() {}) + .to(ResourceSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(RestSampler::class.java) .asEagerSingleton() From a024d0e02ca0bd3a3760510fc2096e2c2b12868f Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 11 Mar 2025 21:10:31 +0100 Subject: [PATCH 34/50] more DI fixes --- .../core/problem/graphql/service/GraphQLBlackBoxModule.kt | 5 +++++ .../core/problem/graphql/service/GraphQLModule.kt | 8 ++++++-- .../org/evomaster/core/problem/rpc/service/RPCModule.kt | 8 ++++++-- .../core/problem/webfrontend/service/WebModule.kt | 4 ++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt index de60c095b6..b8894748ae 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt @@ -4,6 +4,7 @@ import com.google.inject.AbstractModule import com.google.inject.TypeLiteral import org.evomaster.core.output.service.GraphQLTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.remote.service.RemoteController @@ -18,6 +19,10 @@ class GraphQLBlackBoxModule( ): AbstractModule(){ override fun configure() { + + bind(object : TypeLiteral>() {}) + .to(GraphQLSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(GraphQLSampler::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt index 15ac1c71a6..3a8cdab5d3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.service.GraphQLTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.enterprise.service.EnterpriseModule +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.remote.service.RemoteController @@ -22,9 +23,12 @@ class GraphQLModule : EnterpriseModule() { override fun configure() { + bind(object : TypeLiteral>() {}) + .to(GraphQLSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) - .to(GraphQLSampler::class.java) - .asEagerSingleton() + .to(GraphQLSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(GraphQLSampler::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt index 489fe15620..3cf8ebb688 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.service.RPCTestCaseWriter import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.enterprise.service.EnterpriseModule +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.rpc.RPCIndividual import org.evomaster.core.problem.webfrontend.WebIndividual @@ -25,9 +26,12 @@ import org.evomaster.core.search.service.mutator.StructureMutator class RPCModule : EnterpriseModule(){ override fun configure() { + bind(object : TypeLiteral>() {}) + .to(RPCSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) - .to(RPCSampler::class.java) - .asEagerSingleton() + .to(RPCSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(RPCSampler::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt index 8ae848456f..9c9dba8216 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.output.service.WebTestCaseWriter import org.evomaster.core.problem.enterprise.service.EnterpriseModule +import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.webfrontend.WebIndividual @@ -28,6 +29,9 @@ import org.evomaster.core.search.service.mutator.StructureMutator class WebModule: EnterpriseModule() { override fun configure() { + bind(object : TypeLiteral>() {}) + .to(WebSampler::class.java) + .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(WebSampler::class.java) .asEagerSingleton() From a44ed07eeff2a13773402b24bc251dfaad8cbd2b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 Aug 2025 15:43:13 +0200 Subject: [PATCH 35/50] cleaning up --- .../controller/internal/SutController.java | 27 +++++++++--- .../evomaster/client/java/sql/DataRow.java | 4 +- .../sql/heuristic/SqlExpressionEvaluator.java | 6 +-- .../sql/heuristic/TableColumnResolver.java | 4 +- .../sql/internal/QueryResultTransformer.java | 44 ++++++++++--------- .../client/java/sql/internal/SqlHandler.java | 2 +- .../java/sql/internal/SqlNameContext.java | 14 +++--- .../sql/internal/TablesAndColumnsFinder.java | 4 +- .../heuristic/SqlExpressionEvaluatorTest.java | 2 +- .../SqlHeuristicsCalculatorTest.java | 2 +- .../heuristic/TableColumnResolverTest.java | 6 +-- .../sql/internal/ColumnTableAnalyzerTest.java | 4 +- .../internal/QueryResultTransformerTest.java | 6 +-- .../java/sql/internal/SqlHandlerTest.java | 6 +-- .../internal/TablesAndColumnsFinderTest.java | 14 +++--- 15 files changed, 82 insertions(+), 63 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index cef2c2d29a..1f86e052b5 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -519,9 +519,30 @@ public final void cleanAccessedTables(){ try { setExecutingInitSql(true); - // clean accessed tables +// // clean accessed tables +// Set tableDataToInit = null; +// if (!accessedTables.isEmpty()){ +// List tablesToClean = getTablesToClean(accessedTables); +// if (!tablesToClean.isEmpty()){ +// if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ +// emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType, true)); +// } else { +// DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType, true); +// } +// tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); +// emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); +// }else +// DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); +// tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))).collect(Collectors.toSet()); +// +// handleInitSqlInDbClean(tableDataToInit, emDbClean); +// +// } +//// } + Set tableDataToInit = null; if (!accessedTables.isEmpty()){ + //List tablesToClean = new ArrayList<>(); List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ @@ -529,10 +550,6 @@ public final void cleanAccessedTables(){ } else { DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType, true); } - tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> t.equalsIgnoreCase(a))).collect(Collectors.toSet()); - emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType)); - }else - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType); tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))).collect(Collectors.toSet()); } } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java index 7d53be60db..bd3ab135ff 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java @@ -144,8 +144,8 @@ there can be many unnamed tables (eg results of sub-selects) /* We just specified the name without schema... if unique, we would be fine */ - if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ - candidates.add(i); +// if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ +// candidates.add(i); if ((nullSafeEqualsIgnoreCase(n, desc.getColumnName()) || nullSafeEqualsIgnoreCase(n, desc.getAliasColumnName())) && (t == null || t.isEmpty() || nullSafeEqualsIgnoreCase(t, desc.getTableName()) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java index 570e8c08f6..9be451f955 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluator.java @@ -842,9 +842,9 @@ public void visit(ParenthesedSelect parenthesedSelect) { @Override public void visit(Column column) { String name = column.getColumnName(); - String table = sqlNameContext.getFullyQualifiedTableName(column); - Object value = dataRow.getValueByName(name, table); - concreteValues.push(value); +// String table = sqlNameContext.getFullyQualifiedTableName(column); +// Object value = dataRow.getValueByName(name, table); +// concreteValues.push(value); final Object value = getValueForColumn(column); evaluationStack.push(value); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/TableColumnResolver.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/TableColumnResolver.java index 4de6ec8f68..e48a0c9ca1 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/TableColumnResolver.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/heuristic/TableColumnResolver.java @@ -84,7 +84,7 @@ private boolean isBaseTable(String tableName) { Objects.requireNonNull(tableName); return this.schema.tables.stream() - .filter(t -> equalNames(t.name, tableName)) + .filter(t -> equalNames(t.id.name, tableName)) .count() > 0; } @@ -92,7 +92,7 @@ private boolean hasColumn(SqlTableId sqlTableId, SqlColumnId sqlColumnId) { Objects.requireNonNull(sqlTableId); return this.schema.tables.stream() - .filter(t -> new SqlTableId(t.name).equals(sqlTableId)) + .filter(t -> new SqlTableId(t.id.name).equals(sqlTableId)) .flatMap(t -> t.columns.stream()) .filter(c -> new SqlColumnId(c.name).equals(sqlColumnId)) .count() > 0; diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 3a77e1835d..4f09fdf8fa 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -140,12 +140,14 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti List existingQueryResults) { List relatedColumnNames = SqlDatabaseDtoUtils.extractColumnNamesUsedInTheInsertion(insertionDto, relatedColumns); +// List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); +// if (!relatedColumnNames.isEmpty()) { + final QueryResult existingQueryResult; + + //private static QueryResult convertInsertionDtoToQueryResult(InsertionDto insertionDto, String tableName, Set relatedColumns, DbInfoDto dto, List existingQueryResults){ + //List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); if (!relatedColumnNames.isEmpty()) { - final QueryResult existingQueryResult; - private static QueryResult convertInsertionDtoToQueryResult(InsertionDto insertionDto, String tableName, Set relatedColumns, DbInfoDto dto, List existingQueryResults){ - List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); - if (!relatedColumnNames.isEmpty()){ - QueryResult found = null; + //QueryResult found = null; if (!existingQueryResults.isEmpty()) existingQueryResult = existingQueryResults.stream().filter(qr -> qr.sameVariableNames(relatedColumnNames, tableId.getTableId())).findAny().orElse(null); else @@ -158,38 +160,38 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti queryResult = existingQueryResult; Optional foundTableSchema = dto.tables.stream() - .filter(t-> SqlDtoUtils.matchByName(t,tableName)) +// .filter(t -> SqlDtoUtils.matchByName(t, tableName)) + .filter(t -> SqlDtoUtils.matchByName(t, tableId.getTableId())) .findFirst(); - if (foundTableSchema.isPresent()){ - Optional foundTableSchema = dto.tables.stream().filter(t -> t.name.equalsIgnoreCase(tableId.getTableId())).findFirst(); - if (foundTableSchema.isPresent()) { - TableDto tableDto = foundTableSchema.get(); +// if (foundTableSchema.isPresent()) { + // Optional foundTableSchema = dto.tables.stream().filter(t -> t.id.name.equalsIgnoreCase(tableId.getTableId())).findFirst(); + if (foundTableSchema.isPresent()) { + TableDto tableDto = foundTableSchema.get(); - List printableValue = SqlDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); + List printableValue = SqlDatabaseDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); +// List printableValue = SqlDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); assert printableValue.size() == relatedColumnNames.size(); List values = new ArrayList<>(); + //for (int i = 0; i < printableValue.size(); i++) { + // ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); for (int i = 0; i < printableValue.size(); i++) { - ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); - for (int i = 0; i < printableValue.size(); i++){ ColumnDto columnDto = SqlDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); if (columnDto == null) throw new IllegalArgumentException("Cannot find column schema of " + relatedColumnNames.get(i) + " in Table " + tableId); values.add(getColumnValueBasedOnPrintableValue(printableValue.get(i), columnDto)); } - - queryResult.addRow(relatedColumnNames, tableId.getTableId(), values); - } else { throw new IllegalArgumentException("Cannot find table schema of " + tableId); } - if (existingQueryResult != null) return null; - - return queryResult; +// if (existingQueryResult != null) return null; +// +// return queryResult; + //} } return null; } @@ -251,7 +253,7 @@ public static QueryResultSet translateInsertionDtos( QueryResultSet queryResultSet = new QueryResultSet(); for (SqlTableId tableId : columns.keySet()) { TableDto tableDto = schema.tables.stream() - .filter(t -> t.name.equalsIgnoreCase(tableId.getTableId())) + .filter(t -> t.id.name.equalsIgnoreCase(tableId.getTableId())) .findFirst().orElseThrow(() -> new IllegalArgumentException("Cannot find table schema of " + tableId)); List variableDescriptors = tableDto.columns.stream() @@ -281,7 +283,7 @@ private static Object findConcreteValueOrNull(String columnName, TableDto tableD if (entry.variableName.equalsIgnoreCase(columnName)) { ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, entry.variableName); if (columnDto == null) { - throw new IllegalArgumentException("Cannot find column schema of " + entry.variableName + " in Table " + tableDto.name); + throw new IllegalArgumentException("Cannot find column schema of " + entry.variableName + " in Table " + tableDto.id.name); } return getColumnValueBasedOnPrintableValue(entry.printableValue, columnDto); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java index f28bd6df55..0062873679 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlHandler.java @@ -489,7 +489,7 @@ Map> extractColumnsInvolvedInWhere(Statement statem public void visit(Column column) { String tableName = context.getFullyQualifiedTableName(column); - String tableName = context.getTableName(column); +// String tableName = context.getTableName(column); if (tableName.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)) { // TODO handle it properly when ll have support for sub-selects diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java index 9dad0fb7ad..edabbacf62 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlNameContext.java @@ -130,13 +130,13 @@ private List getTableNamesInFrom() { if (hasFromItem()) { FromItem fromItem = getFromItem(); - FromItemVisitorAdapter visitor = new FromItemVisitorAdapter(){ - @Override - public void visit(Table table) { - names.add(table.getFullyQualifiedName().toLowerCase()); - } - }; - FromItemVisitorAdapter visitor = new FromItemVisitorAdapter() { +// FromItemVisitorAdapter visitor = new FromItemVisitorAdapter(){ +// @Override +// public void visit(Table table) { +// names.add(table.getFullyQualifiedName().toLowerCase()); +// } +// }; + FromItemVisitorAdapter visitor = new FromItemVisitorAdapter() { @Override public void visit(Table table) { names.add(table.getName().toLowerCase()); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinder.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinder.java index 4160b1d448..3fa90de9fc 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinder.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinder.java @@ -138,7 +138,7 @@ public void visit(AllTableColumns allTableColumns) { SqlBaseTableReference baseTableReference = (SqlBaseTableReference) tableReference; this.schema.tables.stream() - .filter(t -> new SqlTableId(t.name).equals(baseTableReference.getTableId())) + .filter(t -> new SqlTableId(t.id.name).equals(baseTableReference.getTableId())) .flatMap(t -> t.columns.stream()) .map(c -> new SqlColumnReference(baseTableReference, c.name)) .forEach(c -> addColumnReference(baseTableReference, c)); @@ -176,7 +176,7 @@ private Set findColumnReferences(SqlTableId baseTableId) { Objects.requireNonNull(baseTableId); return this.schema.tables.stream() - .filter(t -> new SqlTableId(t.name).equals(baseTableId)) + .filter(t -> new SqlTableId(t.id.name).equals(baseTableId)) .flatMap(t -> t.columns.stream()) .map(c -> new SqlColumnReference(new SqlBaseTableReference(c.table), c.name)) .collect(Collectors.toSet()); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java index 97f20528f2..8cd99e1db1 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java @@ -44,7 +44,7 @@ private static ColumnDto createColumnDto(String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); - table.name = tableName; + table.id.name = tableName; return table; } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java index bdfe315ddf..d46e1ec86d 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java @@ -398,7 +398,7 @@ private static ColumnDto createColumnDto(String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); - table.name = tableName; + table.id.name = tableName; return table; } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java index 90d6554a72..92ed8adcb3 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java @@ -64,7 +64,7 @@ private static ColumnDto createColumnDto(String tableName, String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); - table.name = tableName; + table.id.name = tableName; return table; } @@ -762,7 +762,7 @@ void testResolveNonNullColumn() throws Exception { void testResolveColumnTableNotInSchema() throws Exception { Assumptions.assumeTrue(this.schema.tables.stream() - .filter(t -> t.name.equals("Foo")) + .filter(t -> t.id.name.equals("Foo")) .count()==0); String sql = "SELECT * FROM Foo"; @@ -782,7 +782,7 @@ void testResolveColumnTableNotInSchema() throws Exception { void testResolveTableNotInSchema() throws Exception { Assumptions.assumeTrue(this.schema.tables.stream() - .filter(t -> t.name.equals("Foo")) + .filter(t -> t.id.name.equals("Foo")) .count()==0); String sql = "SELECT * FROM Foo"; diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java index 152f8bcef3..0bbce9d2d3 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java @@ -59,7 +59,7 @@ public void testDeleteSimpleTable() { DbInfoDto schema = new DbInfoDto(); TableDto fooTableDto = new TableDto(); - fooTableDto.name = "Foo"; + fooTableDto.id.name = "Foo"; schema.tables.add(fooTableDto); SqlTableId deletedTableId = ColumnTableAnalyzer.getDeletedTable(sql); @@ -74,7 +74,7 @@ public void testDeleteWithQualifier() { DbInfoDto schema = new DbInfoDto(); TableDto tableDto = new TableDto(); - tableDto.name = "v1.Foo"; + tableDto.id.name = "v1.Foo"; schema.tables.add(tableDto); SqlTableId deletedTableId = ColumnTableAnalyzer.getDeletedTable(sql); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index f9310a1cbb..a8fe17a29c 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -202,20 +202,20 @@ private static ColumnDto createColumnDto(String tableName, String columnName, St private static DbInfoDto createSchema() { TableDto employeesTable = new TableDto(); - employeesTable.name = "Employees"; + employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("employees", "id", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "name", "VARCHAR")); employeesTable.columns.add(createColumnDto("employees", "income", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "department_id", "INTEGER")); TableDto departmentsTable = new TableDto(); - departmentsTable.name = "Departments"; + departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("departments", "id", "INTEGER")); departmentsTable.columns.add(createColumnDto("departments", "name", "VARCHAR")); departmentsTable.columns.add(createColumnDto("departments", "location_id", "INTEGER")); TableDto locationsTable = new TableDto(); - locationsTable.name = "Locations"; + locationsTable.id.name = "Locations"; locationsTable.columns.add(createColumnDto("locations", "id", "INTEGER")); locationsTable.columns.add(createColumnDto("locations", "city", "VARCHAR")); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java index a645f340dc..4388864b89 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java @@ -176,20 +176,20 @@ private static ColumnDto createColumnDto(String tableName, String columnName, St private static @NotNull DbInfoDto createSchema() { TableDto employeesTable = new TableDto(); - employeesTable.name = "Employees"; + employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("employees", "id", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "name", "VARCHAR")); employeesTable.columns.add(createColumnDto("employees", "income", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "department_id", "INTEGER")); TableDto departmentsTable = new TableDto(); - departmentsTable.name = "Departments"; + departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("departments", "id", "INTEGER")); departmentsTable.columns.add(createColumnDto("departments", "name", "VARCHAR")); departmentsTable.columns.add(createColumnDto("departments", "location_id", "INTEGER")); TableDto locationsTable = new TableDto(); - locationsTable.name = "Locations"; + locationsTable.id.name = "Locations"; locationsTable.columns.add(createColumnDto("locations", "id", "INTEGER")); locationsTable.columns.add(createColumnDto("locations", "city", "VARCHAR")); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java index a3c8714f89..52ef8dbb63 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java @@ -25,34 +25,34 @@ private static DbInfoDto createSchema() { DbInfoDto schema = new DbInfoDto(); TableDto usersTable = new TableDto(); - usersTable.name = "Users"; + usersTable.id.name = "Users"; usersTable.columns.add(createColumnDto("name", "Users")); usersTable.columns.add(createColumnDto("age", "Users")); TableDto employeesTable = new TableDto(); - employeesTable.name = "Employees"; + employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("name", "Employees")); employeesTable.columns.add(createColumnDto("department_id", "Employees")); TableDto departmentsTable = new TableDto(); - departmentsTable.name = "Departments"; + departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("id", "Departments")); departmentsTable.columns.add(createColumnDto("department_name", "Departments")); TableDto votingTable = new TableDto(); - votingTable.name = "voting"; + votingTable.id.name = "voting"; votingTable.columns.add(createColumnDto("expired", "voting")); votingTable.columns.add(createColumnDto("created_at", "voting")); votingTable.columns.add(createColumnDto("group_id", "voting")); TableDto groupsTable = new TableDto(); - groupsTable.name = "groups"; + groupsTable.id.name = "groups"; groupsTable.columns.add(createColumnDto("id", "groups")); groupsTable.columns.add(createColumnDto("voting_duration", "groups")); TableDto dbBaseTable = new TableDto(); - dbBaseTable.name = "db_base"; + dbBaseTable.id.name = "db_base"; dbBaseTable.columns.add(createColumnDto("id", "db_base")); dbBaseTable.columns.add(createColumnDto("name", "db_base")); @@ -622,7 +622,7 @@ void testMissingTables() throws JSQLParserException { DbInfoDto schema = createSchema(); Assumptions.assumeTrue(schema.tables.stream() - .filter(t -> t.name.equals("Foo")) + .filter(t -> t.id.name.equals("Foo")) .count()==0); String sql = "SELECT * FROM Foo"; From f656807be97e0e231a9d5d991cf8979001501d54 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 Aug 2025 15:49:44 +0200 Subject: [PATCH 36/50] cleaning up --- .../rest/resource/dependency/ResourceRelatedToResource.kt | 1 - .../core/problem/rest/service/module/BlackBoxRestModule.kt | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt index c4ce1213da..990b1cd661 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToResource.kt @@ -1,6 +1,5 @@ package org.evomaster.core.problem.rest.resource.dependency -import org.evomaster.core.problem.rest.RestPath import org.evomaster.core.sql.schema.TableId import org.evomaster.core.problem.rest.data.RestPath diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt index cb187114c9..db60d8e048 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt @@ -5,13 +5,9 @@ import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.rest.service.fitness.AbstractRestFitness import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler import org.evomaster.core.problem.rest.service.fitness.BlackBoxRestFitness -import org.evomaster.core.problem.rest.service.fitness.ResourceRestFitness import org.evomaster.core.problem.rest.service.sampler.RestSampler -import org.evomaster.core.output.service.RestTestCaseWriter -import org.evomaster.core.output.service.TestCaseWriter -import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.enterprise.service.EnterpriseSampler -import org.evomaster.core.problem.rest.RestIndividual +import org.evomaster.core.problem.rest.service.sampler.ResourceSampler import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.FitnessFunction From 9939e545878b0d7dc3c5f5eb96a52fb6040e02f9 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 30 Aug 2025 21:01:34 +0200 Subject: [PATCH 37/50] more clean up --- .../problem/rest/service/module/RestModule.kt | 3 +- .../core/problem/rpc/service/RPCModule.kt | 1 - .../problem/webfrontend/service/WebModule.kt | 2 -- .../evomaster/core/solver/SmtLibGenerator.kt | 29 ++++++------------- .../evomaster/core/sql/DbActionUtilsTest.kt | 1 - .../mysql/CreateTableBoundedNumberTest.kt | 1 - .../sql/extract/mysql/CreateTableEnumTest.kt | 1 - 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestModule.kt index 21ae1ce14f..4b380becac 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestModule.kt @@ -35,8 +35,7 @@ class RestModule(private val bindRemote: Boolean = true) : RestBaseModule() { bind(object : TypeLiteral>() {}) .to(RestSampler::class.java) .asEagerSingleton() - .to(RestSampler::class.java) - .asEagerSingleton() + bind(object : TypeLiteral>() {}) .to(RestSampler::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt index 9789d03856..c40d5611eb 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt @@ -6,7 +6,6 @@ import org.evomaster.core.output.service.TestCaseWriter import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.problem.enterprise.service.EnterpriseModule import org.evomaster.core.problem.enterprise.service.EnterpriseSampler -import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.rpc.RPCIndividual import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation diff --git a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt index fced3f20e2..42f5711906 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt @@ -6,8 +6,6 @@ import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.output.service.WebTestCaseWriter import org.evomaster.core.problem.enterprise.service.EnterpriseModule import org.evomaster.core.problem.enterprise.service.EnterpriseSampler -import org.evomaster.core.problem.graphql.GraphQLIndividual -import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.problem.webfrontend.WebIndividual import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation diff --git a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt index f8f22262f7..dd82172235 100644 --- a/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt +++ b/core/src/main/kotlin/org/evomaster/core/solver/SmtLibGenerator.kt @@ -65,10 +65,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Declare constants for each row for (i in 1..numberOfRows) { - smt.addNode( - DeclareConstSMTNode("${table.id.name.lowercase(Locale.getDefault())}$i", dataTypeName) - DeclareConstSMTNode("${table.name.lowercase()}$i", dataTypeName) - ) + smt.addNode(DeclareConstSMTNode("${table.id.name.lowercase()}$i", dataTypeName)) } } } @@ -92,8 +89,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which unique constraints are added. */ private fun appendUniqueConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.id.name.lowercase(Locale.getDefault()) - val tableName = table.name.lowercase() + val tableName = table.id.name.lowercase() for (column in table.columns) { if (column.unique) { val nodes = assertForDistinctField(column.name, tableName) @@ -131,8 +127,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @return The corresponding SMT node. */ private fun parseCheckExpression(table: TableDto, condition: SqlCondition, index: Int): SMTNode { - val visitor = SMTConditionVisitor(table.name.lowercase(), emptyMap(), schema.tables, index) - val visitor = SMTConditionVisitor(table.id.name.lowercase(Locale.getDefault()), emptyMap(), schema.tables, index) + val visitor = SMTConditionVisitor(table.id.name.lowercase(), emptyMap(), schema.tables, index) return condition.accept(visitor, null) as SMTNode } @@ -168,8 +163,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendBooleanConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.id.name.lowercase(Locale.getDefault()) - val tableName = table.name.lowercase() + val tableName = table.id.name.lowercase() for (column in table.columns) { if (column.type.equals("BOOLEAN", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -196,8 +190,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I private fun appendTimestampConstraints(smt: SMTLib) { for (table in schema.tables) { - val tableName = table.id.name.lowercase(Locale.getDefault()) - val tableName = table.name.lowercase() + val tableName = table.id.name.lowercase() for (column in table.columns) { if (column.type.equals("TIMESTAMP", ignoreCase = true)) { val columnName = column.name.uppercase() @@ -235,8 +228,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which primary key constraints are added. */ private fun appendPrimaryKeyConstraints(smt: SMTLib, table: TableDto) { - val tableName = table.id.name.lowercase(Locale.getDefault()) - val tableName = table.name.lowercase() + val tableName = table.id.name.lowercase() val primaryKeys = table.columns.filter { it.primaryKey } for (primaryKey in primaryKeys) { @@ -278,13 +270,11 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I * @param table The table for which foreign key constraints are added. */ private fun appendForeignKeyConstraints(smt: SMTLib, table: TableDto) { - val sourceTableName = table.id.name.lowercase(Locale.getDefault()) - val sourceTableName = table.name.lowercase() + val sourceTableName = table.id.name.lowercase() for (foreignKey in table.foreignKeys) { val referencedTable = findReferencedTable(foreignKey) - val referencedTableName = referencedTable.id.name.lowercase(Locale.getDefault()) - val referencedTableName = referencedTable.name.lowercase() + val referencedTableName = referencedTable.id.name.lowercase() val referencedColumnSelector = findReferencedPKSelector(referencedTable, foreignKey) for (sourceColumn in foreignKey.sourceColumns) { @@ -494,8 +484,7 @@ class SmtLibGenerator(private val schema: DbInfoDto, private val numberOfRows: I // Only add GetValueSMTNode for the mentioned tables for (table in schema.tables) { - val tableNameLower = table.name.lowercase() - val tableNameLower = table.id.name.lowercase(Locale.getDefault()) + val tableNameLower = table.id.name.lowercase() if (tablesMentioned.contains(tableNameLower)) { for (i in 1..numberOfRows) { smt.addNode(GetValueSMTNode("$tableNameLower$i")) diff --git a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt index 706b4edda7..a9134fc075 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/DbActionUtilsTest.kt @@ -1,7 +1,6 @@ package org.evomaster.core.sql import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType -import org.evomaster.core.problem.rest.RestIndividual import org.evomaster.core.sql.schema.Column import org.evomaster.core.sql.schema.ColumnDataType import org.evomaster.core.sql.schema.ForeignKey diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt index 526fe80b3c..3a35aeab4c 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableBoundedNumberTest.kt @@ -8,7 +8,6 @@ import org.evomaster.core.search.gene.numeric.DoubleGene import org.evomaster.core.search.gene.numeric.IntegerGene import org.evomaster.core.search.gene.numeric.LongGene import org.evomaster.core.search.gene.wrapper.NullableGene -import org.evomaster.core.search.gene.optional.NullableGene import org.evomaster.core.sql.schema.TableId import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test diff --git a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt index 9a01b3badb..b3aa6d69dc 100644 --- a/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/sql/extract/mysql/CreateTableEnumTest.kt @@ -3,7 +3,6 @@ package org.evomaster.core.sql.extract.mysql import org.evomaster.client.java.sql.DbInfoExtractor import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.search.gene.collection.EnumGene -import org.evomaster.core.search.gene.optional.NullableGene import org.evomaster.core.sql.schema.TableId import org.evomaster.core.search.gene.wrapper.NullableGene import org.junit.jupiter.api.Assertions.* From be3cad7bf4c29e63d21ff862f478b9e99e2226bd Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 30 Aug 2025 21:42:26 +0200 Subject: [PATCH 38/50] fixing some unit tests --- .../client/java/sql/heuristic/SqlExpressionEvaluatorTest.java | 2 ++ .../client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java | 2 ++ .../client/java/sql/heuristic/TableColumnResolverTest.java | 2 ++ .../client/java/sql/internal/QueryResultTransformerTest.java | 3 +++ 4 files changed, 9 insertions(+) diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java index 8cd99e1db1..e08026c596 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlExpressionEvaluatorTest.java @@ -9,6 +9,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.distance.heuristics.Truthness; import org.evomaster.client.java.sql.DataRow; @@ -44,6 +45,7 @@ private static ColumnDto createColumnDto(String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); + table.id = new TableIdDto(); table.id.name = tableName; return table; } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java index d46e1ec86d..074f2831bd 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/SqlHeuristicsCalculatorTest.java @@ -7,6 +7,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.distance.heuristics.Truthness; import org.evomaster.client.java.distance.heuristics.TruthnessUtils; import org.evomaster.client.java.sql.DataRow; @@ -398,6 +399,7 @@ private static ColumnDto createColumnDto(String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); + table.id = new TableIdDto(); table.id.name = tableName; return table; } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java index 92ed8adcb3..060324173e 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/heuristic/TableColumnResolverTest.java @@ -10,6 +10,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.internal.SqlParserUtils; import org.evomaster.client.java.sql.internal.SqlTableId; import org.junit.jupiter.api.Assumptions; @@ -64,6 +65,7 @@ private static ColumnDto createColumnDto(String tableName, String columnName) { private static TableDto createTableDto(String tableName) { TableDto table = new TableDto(); + table.id = new TableIdDto(); table.id.name = tableName; return table; } diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java index a8fe17a29c..7fcb700155 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/QueryResultTransformerTest.java @@ -202,6 +202,7 @@ private static ColumnDto createColumnDto(String tableName, String columnName, St private static DbInfoDto createSchema() { TableDto employeesTable = new TableDto(); + employeesTable.id = new TableIdDto(); employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("employees", "id", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "name", "VARCHAR")); @@ -209,12 +210,14 @@ private static DbInfoDto createSchema() { employeesTable.columns.add(createColumnDto("employees", "department_id", "INTEGER")); TableDto departmentsTable = new TableDto(); + departmentsTable.id = new TableIdDto(); departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("departments", "id", "INTEGER")); departmentsTable.columns.add(createColumnDto("departments", "name", "VARCHAR")); departmentsTable.columns.add(createColumnDto("departments", "location_id", "INTEGER")); TableDto locationsTable = new TableDto(); + locationsTable.id = new TableIdDto(); locationsTable.id.name = "Locations"; locationsTable.columns.add(createColumnDto("locations", "id", "INTEGER")); locationsTable.columns.add(createColumnDto("locations", "city", "VARCHAR")); From 184bb73e29f19d22910200f8ba00c2be2f7a0a74 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 1 Sep 2025 14:38:07 +0200 Subject: [PATCH 39/50] fixing for tests in sql module --- .../evomaster/client/java/sql/DataRow.java | 31 ++++++------------- .../client/java/sql/QueryResultSet.java | 16 +++++++++- .../sql/internal/QueryResultTransformer.java | 30 ++++++------------ .../sql/internal/SqlDatabaseDtoUtils.java | 22 ++++++------- .../sql/internal/ColumnTableAnalyzerTest.java | 3 ++ .../java/sql/internal/SqlHandlerTest.java | 4 +++ .../internal/TablesAndColumnsFinderTest.java | 7 +++++ 7 files changed, 59 insertions(+), 54 deletions(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java index bd3ab135ff..608f6b3d2b 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DataRow.java @@ -125,11 +125,14 @@ public Object getValueByName(String name, String table) { for (int i = 0; i < variableDescriptors.size(); i++) { VariableDescriptor desc = variableDescriptors.get(i); - if (!n.equalsIgnoreCase(desc.getColumnName())){ + boolean matchColumnName = nullSafeEqualsIgnoreCase(n, desc.getColumnName()) + || nullSafeEqualsIgnoreCase(n, desc.getAliasColumnName()); + + if (!matchColumnName){ continue; } //no defined table, or exact match - if(t == null || t.isEmpty() || t.equalsIgnoreCase(desc.getTableName())){ + if(t == null || t.isEmpty() || nullSafeEqualsIgnoreCase(t, desc.getTableName()) ){ return getValue(i); } /* @@ -138,29 +141,15 @@ there can be many unnamed tables (eg results of sub-selects) with same column names. At this moment, we would not be able to distinguish them */ - if(t.equalsIgnoreCase(SqlNameContext.UNNAMED_TABLE)){ + if(nullSafeEqualsIgnoreCase(t, SqlNameContext.UNNAMED_TABLE)){ candidates.add(i); } - /* - We just specified the name without schema... if unique, we would be fine - */ -// if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ -// candidates.add(i); - if ((nullSafeEqualsIgnoreCase(n, desc.getColumnName()) || nullSafeEqualsIgnoreCase(n, desc.getAliasColumnName())) && - (t == null || t.isEmpty() - || nullSafeEqualsIgnoreCase(t, desc.getTableName()) - /* - TODO: this does not cover all possible cases, as in theory - there can be many unnamed tables (eg results of sub-selects) - with same column names. At this moment, we would not - be able to distinguish them - */ - || nullSafeEqualsIgnoreCase(t, SqlNameContext.UNNAMED_TABLE) - ) - ) { - return getValue(i); + + if(!t.contains(".") && desc.getTableName().toLowerCase().endsWith("."+t.toLowerCase())){ + candidates.add(i); } } + if(candidates.size() > 1){ SimpleLogger.uniqueWarn("More than one table candidate for: " + t); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResultSet.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResultSet.java index 0256f8da05..c9ca19322c 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResultSet.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/QueryResultSet.java @@ -93,7 +93,21 @@ private void handleNamedTable(String tableName, QueryResult queryResult) { public QueryResult getQueryResultForNamedTable(String tableName) { Objects.requireNonNull(tableName); - return queryResults.get(tableName.toLowerCase()); + + + String name = tableName.toLowerCase(); + + QueryResult res = queryResults.get(name); + if(res != null){ + return res; + } + + //FIXME should do proper handling of schema/catalog + return queryResults.entrySet().stream() + .filter(it -> it.getKey().toLowerCase().endsWith("."+name)) + .findFirst() + .map(Map.Entry::getValue) + .orElse(null); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java index 4f09fdf8fa..f5fe2838ae 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/QueryResultTransformer.java @@ -140,14 +140,10 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti List existingQueryResults) { List relatedColumnNames = SqlDatabaseDtoUtils.extractColumnNamesUsedInTheInsertion(insertionDto, relatedColumns); -// List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); -// if (!relatedColumnNames.isEmpty()) { - final QueryResult existingQueryResult; - //private static QueryResult convertInsertionDtoToQueryResult(InsertionDto insertionDto, String tableName, Set relatedColumns, DbInfoDto dto, List existingQueryResults){ - //List relatedColumnNames = SqlDtoUtils.extractColumnNames(insertionDto, relatedColumns); if (!relatedColumnNames.isEmpty()) { - //QueryResult found = null; + + final QueryResult existingQueryResult; if (!existingQueryResults.isEmpty()) existingQueryResult = existingQueryResults.stream().filter(qr -> qr.sameVariableNames(relatedColumnNames, tableId.getTableId())).findAny().orElse(null); else @@ -160,23 +156,17 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti queryResult = existingQueryResult; Optional foundTableSchema = dto.tables.stream() -// .filter(t -> SqlDtoUtils.matchByName(t, tableName)) .filter(t -> SqlDtoUtils.matchByName(t, tableId.getTableId())) .findFirst(); -// if (foundTableSchema.isPresent()) { - // Optional foundTableSchema = dto.tables.stream().filter(t -> t.id.name.equalsIgnoreCase(tableId.getTableId())).findFirst(); - if (foundTableSchema.isPresent()) { - TableDto tableDto = foundTableSchema.get(); + if (foundTableSchema.isPresent()) { + TableDto tableDto = foundTableSchema.get(); - List printableValue = SqlDatabaseDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); -// List printableValue = SqlDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); + List printableValue = SqlDatabaseDtoUtils.extractColumnPrintableValues(insertionDto, relatedColumns); assert printableValue.size() == relatedColumnNames.size(); List values = new ArrayList<>(); - //for (int i = 0; i < printableValue.size(); i++) { - // ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); for (int i = 0; i < printableValue.size(); i++) { ColumnDto columnDto = SqlDtoUtils.extractColumnInfo(tableDto, relatedColumnNames.get(i)); if (columnDto == null) @@ -184,14 +174,14 @@ private static QueryResult convertInsertionDtoToQueryResult(InsertionDto inserti values.add(getColumnValueBasedOnPrintableValue(printableValue.get(i), columnDto)); } queryResult.addRow(relatedColumnNames, tableId.getTableId(), values); + } else { throw new IllegalArgumentException("Cannot find table schema of " + tableId); } -// if (existingQueryResult != null) return null; -// -// return queryResult; - //} + if (existingQueryResult != null) return null; + + return queryResult; } return null; } @@ -281,7 +271,7 @@ public static QueryResultSet translateInsertionDtos( private static Object findConcreteValueOrNull(String columnName, TableDto tableDto, List entries) { for (InsertionEntryDto entry : entries) { if (entry.variableName.equalsIgnoreCase(columnName)) { - ColumnDto columnDto = SqlDatabaseDtoUtils.extractColumnInfo(tableDto, entry.variableName); + ColumnDto columnDto = SqlDtoUtils.extractColumnInfo(tableDto, entry.variableName); if (columnDto == null) { throw new IllegalArgumentException("Cannot find column schema of " + entry.variableName + " in Table " + tableDto.id.name); } diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java index 4d1a00457d..bdeea03c8c 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/internal/SqlDatabaseDtoUtils.java @@ -1,6 +1,7 @@ package org.evomaster.client.java.sql.internal; +import org.evomaster.client.java.controller.api.dto.SqlDtoUtils; import org.evomaster.client.java.controller.api.dto.database.operations.InsertionDto; import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; @@ -10,6 +11,9 @@ import java.util.Set; import java.util.stream.Collectors; +/** + * FIXME this will need refactoring + */ public class SqlDatabaseDtoUtils { @@ -19,7 +23,9 @@ public class SqlDatabaseDtoUtils { * @return name of columns based on specified filter */ public static List extractColumnNamesUsedInTheInsertion(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f.getColumnId()))).map(i-> i.variableName).collect(Collectors.toList()); + + return SqlDtoUtils.extractColumnNames(dto, filter.stream().map(it -> it.getColumnId()).collect(Collectors.toSet())); + //return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f.getColumnId()))).map(i-> i.variableName).collect(Collectors.toList()); } /** @@ -28,17 +34,9 @@ public static List extractColumnNamesUsedInTheInsertion(InsertionDto dto * @return printable value of columns based on specified filter */ public static List extractColumnPrintableValues(InsertionDto dto, Set filter){ - return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f.getColumnId()))).map(i-> i.printableValue).collect(Collectors.toList()); - } - - /** - * - * @param columnName specified which ColumnDto should be returned based on its name - * @return ColumnDto based on specified columnName - */ - public static ColumnDto extractColumnInfo(TableDto dto, String columnName){ - Optional op = dto.columns.stream().filter(c-> columnName.equalsIgnoreCase(c.name)).findAny(); - return op.orElse(null); + return SqlDtoUtils.extractColumnPrintableValues(dto, filter.stream().map(it -> it.getColumnId()).collect(Collectors.toSet())); + //return dto.data.stream().filter(i-> (filter == null) || filter.stream().anyMatch(f-> i.variableName.equalsIgnoreCase(f.getColumnId()))).map(i-> i.printableValue).collect(Collectors.toList()); } + } \ No newline at end of file diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java index 0bbce9d2d3..78b009c578 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/ColumnTableAnalyzerTest.java @@ -3,6 +3,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; @@ -59,6 +60,7 @@ public void testDeleteSimpleTable() { DbInfoDto schema = new DbInfoDto(); TableDto fooTableDto = new TableDto(); + fooTableDto.id = new TableIdDto(); fooTableDto.id.name = "Foo"; schema.tables.add(fooTableDto); SqlTableId deletedTableId = ColumnTableAnalyzer.getDeletedTable(sql); @@ -74,6 +76,7 @@ public void testDeleteWithQualifier() { DbInfoDto schema = new DbInfoDto(); TableDto tableDto = new TableDto(); + tableDto.id = new TableIdDto(); tableDto.id.name = "v1.Foo"; schema.tables.add(tableDto); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java index 4388864b89..82bd782728 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/SqlHandlerTest.java @@ -7,6 +7,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -176,6 +177,7 @@ private static ColumnDto createColumnDto(String tableName, String columnName, St private static @NotNull DbInfoDto createSchema() { TableDto employeesTable = new TableDto(); + employeesTable.id = new TableIdDto(); employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("employees", "id", "INTEGER")); employeesTable.columns.add(createColumnDto("employees", "name", "VARCHAR")); @@ -183,12 +185,14 @@ private static ColumnDto createColumnDto(String tableName, String columnName, St employeesTable.columns.add(createColumnDto("employees", "department_id", "INTEGER")); TableDto departmentsTable = new TableDto(); + departmentsTable.id = new TableIdDto(); departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("departments", "id", "INTEGER")); departmentsTable.columns.add(createColumnDto("departments", "name", "VARCHAR")); departmentsTable.columns.add(createColumnDto("departments", "location_id", "INTEGER")); TableDto locationsTable = new TableDto(); + locationsTable.id = new TableIdDto(); locationsTable.id.name = "Locations"; locationsTable.columns.add(createColumnDto("locations", "id", "INTEGER")); locationsTable.columns.add(createColumnDto("locations", "city", "VARCHAR")); diff --git a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java index 52ef8dbb63..58b2e71e0e 100644 --- a/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java +++ b/client-java/sql/src/test/java/org/evomaster/client/java/sql/internal/TablesAndColumnsFinderTest.java @@ -6,6 +6,7 @@ import org.evomaster.client.java.controller.api.dto.database.schema.ColumnDto; import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto; import org.evomaster.client.java.controller.api.dto.database.schema.TableDto; +import org.evomaster.client.java.controller.api.dto.database.schema.TableIdDto; import org.evomaster.client.java.sql.heuristic.SqlColumnReference; import org.evomaster.client.java.sql.heuristic.SqlBaseTableReference; import org.jetbrains.annotations.NotNull; @@ -25,33 +26,39 @@ private static DbInfoDto createSchema() { DbInfoDto schema = new DbInfoDto(); TableDto usersTable = new TableDto(); + usersTable.id = new TableIdDto(); usersTable.id.name = "Users"; usersTable.columns.add(createColumnDto("name", "Users")); usersTable.columns.add(createColumnDto("age", "Users")); TableDto employeesTable = new TableDto(); + employeesTable.id = new TableIdDto(); employeesTable.id.name = "Employees"; employeesTable.columns.add(createColumnDto("name", "Employees")); employeesTable.columns.add(createColumnDto("department_id", "Employees")); TableDto departmentsTable = new TableDto(); + departmentsTable.id = new TableIdDto(); departmentsTable.id.name = "Departments"; departmentsTable.columns.add(createColumnDto("id", "Departments")); departmentsTable.columns.add(createColumnDto("department_name", "Departments")); TableDto votingTable = new TableDto(); + votingTable.id = new TableIdDto(); votingTable.id.name = "voting"; votingTable.columns.add(createColumnDto("expired", "voting")); votingTable.columns.add(createColumnDto("created_at", "voting")); votingTable.columns.add(createColumnDto("group_id", "voting")); TableDto groupsTable = new TableDto(); + groupsTable.id = new TableIdDto(); groupsTable.id.name = "groups"; groupsTable.columns.add(createColumnDto("id", "groups")); groupsTable.columns.add(createColumnDto("voting_duration", "groups")); TableDto dbBaseTable = new TableDto(); + dbBaseTable.id = new TableIdDto(); dbBaseTable.id.name = "db_base"; dbBaseTable.columns.add(createColumnDto("id", "db_base")); dbBaseTable.columns.add(createColumnDto("name", "db_base")); From 2381917a6270c459f4fc3ae54053088d7b713e85 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 1 Sep 2025 21:20:40 +0200 Subject: [PATCH 40/50] fixed some compilation error --- .../spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt index baa1309fb5..3e8faa3ea5 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/separatedschemas/SeparatedSchemasEMTest.kt @@ -1,8 +1,7 @@ package org.evomaster.e2etests.spring.multidb.separatedschemas import org.evomaster.client.java.controller.api.dto.database.schema.DatabaseType -import org.evomaster.client.java.sql.SqlScriptRunner -import org.evomaster.core.problem.rest.HttpVerb +import org.evomaster.core.problem.rest.data.HttpVerb import org.evomaster.driver.multidb.SpringController import org.evomaster.e2etests.spring.multidb.MultiDbParameterizedE2ETemplate import org.junit.jupiter.api.Assertions.assertTrue From 8359d25b6079b672ea2c075d5b1846e7695a7c8b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 5 Sep 2025 13:51:24 +0200 Subject: [PATCH 41/50] fixed bug in handling of db reset --- .../java/org/evomaster/client/java/sql/DbCleaner.java | 4 ++-- .../spring/examples/db/base/DbBaseManualTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index 007233ee34..9604ed9611 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -26,13 +26,13 @@ public static void clearTables(Connection connection, List fullyQualifyi for(String t : fullyQualifyingTableNames) { if(! t.contains(".")){ - Set x = schemaToNames.getOrDefault(defaultSchema, new HashSet<>()); + Set x = schemaToNames.computeIfAbsent(defaultSchema, it -> new HashSet<>()); x.add(t); } else { String[] tokens = t.split("\\."); String name = tokens[tokens.length - 1]; String schema = tokens[tokens.length - 2]; - Set x = schemaToNames.getOrDefault(schema, new HashSet<>()); + Set x = schemaToNames.computeIfAbsent(schema, it -> new HashSet<>()); x.add(name); } } diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/base/DbBaseManualTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/base/DbBaseManualTest.java index f8c46fbe1f..779751a8ad 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/base/DbBaseManualTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/base/DbBaseManualTest.java @@ -13,6 +13,10 @@ public class DbBaseManualTest extends DbBaseTestBase { + @BeforeEach + public void reset(){ + controller.resetDatabase(Collections.singletonList("db_base_entity")); + } @Test public void testCreateOne() { @@ -93,9 +97,5 @@ public void testGetByName(){ .body("size()", is(2)); } - @BeforeEach - public void reset(){ - controller.resetDatabase(Collections.singletonList("db_base_entity")); - } } From a648626b312476b4cbf4e6d5d05abd338e38fd3b Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 5 Sep 2025 21:50:15 +0200 Subject: [PATCH 42/50] fixed bug after change of semantics --- .../core/problem/enterprise/service/EnterpriseSampler.kt | 4 +++- .../core/problem/rest/service/sampler/ResourceSampler.kt | 4 +++- .../evomaster/e2etests/spring/graphql/db/base/DbBaseEMTest.kt | 3 +-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 9bf7459f46..f7e6eadee2 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -130,7 +130,9 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { } open fun initSqlInfo(infoDto: SutInfoDto) { - if (infoDto.sqlSchemaDto != null && config.shouldGenerateSqlData()) { + if (infoDto.sqlSchemaDto != null + //&& config.shouldGenerateSqlData() //might need even if no insertion, eg, for table names + ) { sqlInsertBuilder = SqlInsertBuilder(infoDto.sqlSchemaDto, rc) existingSqlData = sqlInsertBuilder!!.extractExistingPKs() } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/ResourceSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/ResourceSampler.kt index 2d7d0c7924..74bcda3f3e 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/ResourceSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/ResourceSampler.kt @@ -42,7 +42,9 @@ open class ResourceSampler : AbstractRestSampler() { override fun initSqlInfo(infoDto: SutInfoDto) { //when ResourceDependency is enabled, SQL info is required to identify dependency - if (infoDto.sqlSchemaDto != null && (configuration.shouldGenerateSqlData() || config.isEnabledResourceDependency())) { + if (infoDto.sqlSchemaDto != null + //&& (configuration.shouldGenerateSqlData() || config.isEnabledResourceDependency())//might need even if no insertion, eg, for table names + ) { sqlInsertBuilder = SqlInsertBuilder(infoDto.sqlSchemaDto, rc) existingSqlData = sqlInsertBuilder!!.extractExistingPKs() diff --git a/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/base/DbBaseEMTest.kt b/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/base/DbBaseEMTest.kt index 0f2ebfb8ed..d557a28992 100644 --- a/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/base/DbBaseEMTest.kt +++ b/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/base/DbBaseEMTest.kt @@ -27,8 +27,7 @@ class DbBaseEMTest : SpringTestBase() { @ValueSource(booleans = [false, true]) fun testRunEM(heuristicsForSQLAdvanced: Boolean) { runTestHandlingFlakyAndCompilation( - "GQL_DbBaseEM", - "org.foo.graphql.DbBaseEM" + + "DbBaseEM" + if (heuristicsForSQLAdvanced) "Complete" else "Partial", 10000 ) { args: MutableList -> From 420a05591a19339d4386a6759b9a0673c9904881 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 8 Sep 2025 10:07:24 +0200 Subject: [PATCH 43/50] fix for too restrictive/brittle assertions --- .../e2etests/utils/EnterpriseTestBase.java | 23 ++++++++++++------- .../graphql/db/directint/DbDirectIntEMTest.kt | 7 +++--- .../db/directint/DbDirectIntEMTest.java | 3 ++- .../db/directint/DbDirectIntEMTest.java | 3 ++- .../db/directint/DbDirectIntEMTest.java | 3 ++- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 045ab5dc11..5d011a2303 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -44,6 +44,8 @@ import java.util.List; import java.util.Objects; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; import static org.junit.jupiter.api.Assertions.*; @@ -553,18 +555,23 @@ protected void setOption(List args, String optionName, String optionValu } } - /** - * assert a certain text in the generated tests - * @param outputFolder the folder where the test is - * @param className the complete test name - * @param content is the content to check - */ + protected void assertTextInTests(String outputFolder, String className, String content) { + assertTextInTests(outputFolder,className, l -> l.contains(content)); + } + + /** + * assert a certain text in the generated tests + * @param outputFolder the folder where the test is + * @param className the complete test name + * @param condition is the content to check + */ + protected void assertTextInTests(String outputFolder, String className, Predicate condition) { String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt"; Path test = Paths.get(path); try { - boolean ok = Files.lines(test).anyMatch(l-> l.contains(content)); - String msg = "Cannot find "+content+" in "+className+" in "+outputFolder; + boolean ok = Files.lines(test).anyMatch(condition); + String msg = "Cannot find line with requested condition in "+className+" in "+outputFolder; assertTrue(ok, msg); }catch (IOException e){ throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); diff --git a/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/directint/DbDirectIntEMTest.kt b/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/directint/DbDirectIntEMTest.kt index 6c1070fbbe..f019f4bcdc 100644 --- a/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/directint/DbDirectIntEMTest.kt +++ b/e2e-tests/spring-graphql/src/test/kotlin/org/evomaster/e2etests/spring/graphql/db/directint/DbDirectIntEMTest.kt @@ -1,6 +1,7 @@ package org.evomaster.e2etests.spring.graphql.db.directint import com.foo.graphql.db.directint.DbDirectIntController +import graphql.com.google.common.base.Predicate import org.evomaster.core.EMConfig import org.evomaster.core.problem.graphql.GQMethodType import org.evomaster.e2etests.spring.graphql.SpringTestBase @@ -72,9 +73,9 @@ class DbDirectIntEMTest : SpringTestBase() { assertHasAtLeastOne(solution, "get", GQMethodType.QUERY, 200, "\"get\":[]") assertNoneWithErrors(solution) - assertTextInTests(outputFolder, outputTestName, - "controller.resetDatabase(listOf(\"db_direct_int\"))" - ) + assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase(") + val condition: Predicate = Predicate { it!!.contains("db_direct_int", true) } + assertTextInTests(outputFolder, outputTestName, condition) } } } diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java index 2e31d6e46e..d4034223b8 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java @@ -55,7 +55,8 @@ private void testRunEM(EMConfig.SecondaryObjectiveStrategy strategy, boolean heu assertHasAtLeastOne(solution, HttpVerb.POST, 200, "/api/db/directint", null); assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/db/directint/{x}/{y}", null); - assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase(listOf(\"db_direct_int_entity\"))"); + assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase("); + assertTextInTests(outputFolder, outputTestName, it -> it.toLowerCase().contains("db_direct_int_entity")); }); } } diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java index d618c81ef5..3efe7290f3 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directint/DbDirectIntEMTest.java @@ -55,7 +55,8 @@ private void testRunEM(EMConfig.SecondaryObjectiveStrategy strategy, boolean heu assertHasAtLeastOne(solution, HttpVerb.POST, 200, "/api/db/directint", null); assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/db/directint/{x}/{y}", null); - assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase(listOf(\"db_direct_int_entity\"))"); + assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase("); + assertTextInTests(outputFolder, outputTestName, it -> it.toLowerCase().contains("db_direct_int_entity")); }); } } diff --git a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/db/directint/DbDirectIntEMTest.java b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/db/directint/DbDirectIntEMTest.java index cecea76ff6..d69fcbb26d 100644 --- a/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/db/directint/DbDirectIntEMTest.java +++ b/e2e-tests/spring-rpc/spring-rpc-thrift/src/test/java/org/evomaster/e2etests/spring/rpc/examples/db/directint/DbDirectIntEMTest.java @@ -64,7 +64,8 @@ private void testRunEM(EMConfig.SecondaryObjectiveStrategy strategy, boolean heu assertContentInResponseForEndpoint(solution, DbDirectIntService.Iface.class.getName()+":get", "400"); assertContentInResponseForEndpoint(solution, DbDirectIntService.Iface.class.getName()+":get", "200"); - assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase(listOf(\"db_direct_int_entity\"))"); + assertTextInTests(outputFolder, outputTestName, "controller.resetDatabase("); + assertTextInTests(outputFolder, outputTestName, it -> it.toLowerCase().contains("db_direct_int_entity")); }); } } From 1840d05ade3bee0ba06de81372dbf8a2ab048460 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 9 Sep 2025 20:55:10 +0200 Subject: [PATCH 44/50] tmp hack to deal with issue in resetting SQL tables --- .../controller/internal/SutController.java | 21 ++++++++++++++++--- .../org/evomaster/core/output/SqlWriter.kt | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 1f86e052b5..c8293a3b73 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -101,6 +101,8 @@ public abstract class SutController implements SutHandler, CustomizationHandler /** * track all tables accessed in a test + * + * FIXME shouldn't use strings */ private final List accessedTables = new CopyOnWriteArrayList<>(); @@ -112,12 +114,16 @@ public abstract class SutController implements SutHandler, CustomizationHandler /** * a map of table to fk target tables + * + * FIXME shouldn't use strings */ private final Map> fkMap = new ConcurrentHashMap<>(); /** * a map of table to a set of commands which are to insert data into the db + * + * FIXME shouldn't use strings */ private final Map> tableInitSqlMap = new ConcurrentHashMap<>(); @@ -545,10 +551,18 @@ public final void cleanAccessedTables(){ //List tablesToClean = new ArrayList<>(); List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ + /* + TODO should emDbClean.schemaNames be deprecated/removed? + */ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType, true)); } else { - DbCleaner.clearDatabase(getConnectionIfExist(), null, null, tablesToClean, emDbClean.dbType, true); + //FIXME ignoring schema here + List names = tablesToClean.stream().map(it -> { + String [] tokens = it.split("\\."); + return tokens[tokens.length - 1]; + }).collect(Collectors.toList()); + DbCleaner.clearDatabase(getConnectionIfExist(), null, null, names, emDbClean.dbType, true); } tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))).collect(Collectors.toSet()); } @@ -766,10 +780,11 @@ public final boolean extractSqlDbSchemaAndConstraints(){ if (fkMap.isEmpty()){ schemaDto.tables.forEach(t->{ - fkMap.putIfAbsent(t.id.name, new ArrayList<>()); + String id = SqlDtoUtils.getId(t); + fkMap.putIfAbsent(id, new ArrayList<>()); if (t.foreignKeys!=null && !t.foreignKeys.isEmpty()){ t.foreignKeys.forEach(f->{ - fkMap.get(t.id.name).add(f.targetTable.toUpperCase()); + fkMap.get(id).add(f.targetTable.toUpperCase()); }); } }); diff --git a/core/src/main/kotlin/org/evomaster/core/output/SqlWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/SqlWriter.kt index c74faa3716..08da985ae7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/SqlWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/SqlWriter.kt @@ -36,7 +36,8 @@ object SqlWriter { insertionVars: MutableList>, skipFailure: Boolean) { - if (dbInitialization.isEmpty() || dbInitialization.none { !it.sqlAction.representExistingData && (!skipFailure || it.sqlResult.getInsertExecutionResult())}) { + if (dbInitialization.isEmpty() + || dbInitialization.none { !it.sqlAction.representExistingData && (!skipFailure || it.sqlResult.getInsertExecutionResult())}) { return } From 0891bf2cb5d2ac049ddd2ac79a665d8676b97c05 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 23 Sep 2025 22:52:09 +0200 Subject: [PATCH 45/50] dirty hacks to fix some of the tests, before further major refactoring --- .../controller/internal/SutController.java | 46 ++++++++++++------- .../dto/database/operations/InsertionDto.java | 3 ++ .../evomaster/client/java/sql/DbCleaner.java | 20 ++++++-- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 4dc5795161..26a62332c7 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -105,7 +105,7 @@ public abstract class SutController implements SutHandler, CustomizationHandler /** * track all tables accessed in a test * - * FIXME shouldn't use strings + * FIXME shouldn't use string SqlTableId */ private final List accessedTables = new CopyOnWriteArrayList<>(); @@ -118,7 +118,7 @@ public abstract class SutController implements SutHandler, CustomizationHandler /** * a map of table to fk target tables * - * FIXME shouldn't use strings + * FIXME shouldn't use string but SqlTableId */ private final Map> fkMap = new ConcurrentHashMap<>(); @@ -126,7 +126,7 @@ public abstract class SutController implements SutHandler, CustomizationHandler /** * a map of table to a set of commands which are to insert data into the db * - * FIXME shouldn't use strings + * FIXME shouldn't use string SqlTableId */ private final Map> tableInitSqlMap = new ConcurrentHashMap<>(); @@ -457,6 +457,7 @@ private void computeSQLHeuristics(ExtraHeuristicsDto dto, List a if (sqlExecutionsDto != null) { accessedTables.addAll(sqlExecutionsDto.deletedData); accessedTables.addAll(sqlExecutionsDto.insertedData.keySet()); + //TODO should accessedTables be renamed into modifiedTables??? //accessedTables.addAll(executionDto.queriedData.keySet()); accessedTables.addAll(sqlExecutionsDto.updatedData.keySet()); } @@ -716,20 +717,33 @@ private List getTablesToClean(List accessedTables) { return tablesToClean; } - private void fillTablesToClean(List accessedTables, List tablesToClean){ - for (String t: accessedTables){ - if (!findInCollectionIgnoreCase(t, tablesToClean).isPresent()){ - if (findInMapIgnoreCase(t, fkMap).isPresent()){ - tablesToClean.add(t); - List fk = fkMap.entrySet().stream().filter(e-> - findInCollectionIgnoreCase(t, e.getValue()).isPresent() - && !findInCollectionIgnoreCase(e.getKey(), tablesToClean).isPresent()).map(Map.Entry::getKey).collect(Collectors.toList()); - if (!fk.isEmpty()) - fillTablesToClean(fk, tablesToClean); - }else { - SimpleLogger.uniqueWarn("Cannot find the table "+t+" in ["+String.join(",", fkMap.keySet())+"]"); - } + private boolean hasTableNameDirtyHack(String name, Collection tableIds){ + //FIXME when refactoring datastructures in this class, remove + return tableIds.stream().anyMatch(i-> i.toLowerCase().endsWith(name.toLowerCase())); + } + + private void fillTablesToClean(List accessedTables, List tablesToClean){ + for (String t : accessedTables) { + if (findInCollectionIgnoreCase(t, tablesToClean).isPresent()) { + continue; + } + //if (findInMapIgnoreCase(t, fkMap).isPresent()) { + if(hasTableNameDirtyHack(t,fkMap.keySet())){ + tablesToClean.add(t); + List fk = fkMap.entrySet().stream() + .filter(e -> + hasTableNameDirtyHack(t, e.getValue()) + && !hasTableNameDirtyHack(e.getKey(), tablesToClean) +// findInCollectionIgnoreCase(t, e.getValue()).isPresent() +// && !findInCollectionIgnoreCase(e.getKey(), tablesToClean).isPresent()) + ) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if (!fk.isEmpty()) + fillTablesToClean(fk, tablesToClean); + } else { + SimpleLogger.uniqueWarn("Cannot find the table " + t + " in [" + String.join(",", fkMap.keySet()) + "]"); } } } diff --git a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/operations/InsertionDto.java b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/operations/InsertionDto.java index b9b1e9ba0e..2cc1667ea4 100644 --- a/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/operations/InsertionDto.java +++ b/client-java/sql-dto/src/main/java/org/evomaster/client/java/controller/api/dto/database/operations/InsertionDto.java @@ -16,6 +16,9 @@ public class InsertionDto { */ public Long id; + /** + * FIXME schema + */ public String targetTable; public List data = new ArrayList<>(); diff --git a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java index 9604ed9611..3f5435ff74 100644 --- a/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java +++ b/client-java/sql/src/main/java/org/evomaster/client/java/sql/DbCleaner.java @@ -223,7 +223,7 @@ private static List cleanDataInTables(List tableToSkip, throw new IllegalStateException("Could not find any table"); } - final List tableToHandle; + List tableToHandle; boolean toskip = tableToSkip != null; if (tableToClean != null) { tableToHandle = tableToClean; @@ -231,6 +231,17 @@ private static List cleanDataInTables(List tableToSkip, tableToHandle = tableToSkip; } + /* + FIXME need refactoring, this is a dirty hack + */ + if(tableToHandle != null) { + tableToHandle = tableToHandle.stream() + .map(t -> { + String[] tokens = t.split("\\."); + return tokens[tokens.length - 1]; + }) + .collect(Collectors.toList()); + } if (tableToHandle != null) { for (String skip : tableToHandle) { @@ -252,10 +263,11 @@ private static List cleanDataInTables(List tableToSkip, rst.close(); } + final List tth = tableToHandle; List tablesToClear = tables.stream() - .filter(n -> tableToHandle == null || - (toskip && (tableToHandle.isEmpty() || tableToHandle.stream().noneMatch(skip -> skip.equalsIgnoreCase(n)))) || - (!toskip && tableToHandle.stream().anyMatch(clean -> clean.equalsIgnoreCase(n))) + .filter(n -> tth == null || + (toskip && (tth.isEmpty() || tth.stream().noneMatch(skip -> skip.equalsIgnoreCase(n)))) || + (!toskip && tth.stream().anyMatch(clean -> clean.equalsIgnoreCase(n))) ) .collect(Collectors.toList()); From 7ee57471a475f52ab254e3614fb43bfa77d654d8 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 25 Sep 2025 14:28:09 +0200 Subject: [PATCH 46/50] dirty hacks for SQL init --- .../controller/internal/SutController.java | 19 +++++++++++++------ .../InitSqlScriptWithSmartDbCleanTest.java | 12 ++++++++---- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index 26a62332c7..fcf40f8780 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -603,20 +603,27 @@ public final void cleanAccessedTables(){ //List tablesToClean = new ArrayList<>(); List tablesToClean = getTablesToClean(accessedTables); if (!tablesToClean.isEmpty()){ + + //FIXME ignoring schema here + List names = tablesToClean.stream().map(it -> { + String [] tokens = it.split("\\."); + return tokens[tokens.length - 1]; + }).collect(Collectors.toList()); + /* TODO should emDbClean.schemaNames be deprecated/removed? */ if (emDbClean.schemaNames != null && !emDbClean.schemaNames.isEmpty()){ emDbClean.schemaNames.forEach(sch-> DbCleaner.clearDatabase(getConnectionIfExist(), sch, null, tablesToClean, emDbClean.dbType, true)); } else { - //FIXME ignoring schema here - List names = tablesToClean.stream().map(it -> { - String [] tokens = it.split("\\."); - return tokens[tokens.length - 1]; - }).collect(Collectors.toList()); DbCleaner.clearDatabase(getConnectionIfExist(), null, null, names, emDbClean.dbType, true); } - tableDataToInit = tablesToClean.stream().filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))).collect(Collectors.toSet()); + + // tableDataToInit = tablesToClean.stream() + //FIXME + tableDataToInit = names.stream() + .filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))) + .collect(Collectors.toSet()); } } handleInitSqlInDbClean(tableDataToInit, emDbClean); diff --git a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/InitSqlScriptWithSmartDbCleanTest.java b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/InitSqlScriptWithSmartDbCleanTest.java index b9f8384d80..ed89f0d932 100644 --- a/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/InitSqlScriptWithSmartDbCleanTest.java +++ b/client-java/controller/src/test/java/org/evomaster/client/java/controller/internal/db/sql/InitSqlScriptWithSmartDbCleanTest.java @@ -39,13 +39,17 @@ default String getInitSqlScript() { @Test default void testAccessedFkClean() throws Exception { - EMSqlScriptRunner.execCommand(getConnection(), "CREATE TABLE Bar(id INT Primary Key, valueColumn INT)", true); - EMSqlScriptRunner.execCommand(getConnection(), "CREATE TABLE Foo(id INT Primary Key, valueColumn INT, refer_foo INT DEFAULT NULL, bar_id INT, " + + EMSqlScriptRunner.execCommand(getConnection(), + "CREATE TABLE Bar(id INT Primary Key, valueColumn INT)", true); + EMSqlScriptRunner.execCommand(getConnection(), + "CREATE TABLE Foo(id INT Primary Key, valueColumn INT, refer_foo INT DEFAULT NULL, bar_id INT, " + "CONSTRAINT fk_foo FOREIGN KEY (bar_id) REFERENCES Bar(id)," + "CONSTRAINT fk_self_foo FOREIGN KEY (refer_foo) REFERENCES Foo(id) )", true); - EMSqlScriptRunner.execCommand(getConnection(), "CREATE TABLE Abc(id INT Primary Key, valueColumn INT, foo_id INT, " + + EMSqlScriptRunner.execCommand(getConnection(), + "CREATE TABLE Abc(id INT Primary Key, valueColumn INT, foo_id INT, " + "CONSTRAINT fk_abc FOREIGN KEY (foo_id) REFERENCES Foo(id) )", true); - EMSqlScriptRunner.execCommand(getConnection(), "CREATE TABLE Xyz(id INT Primary Key, valueColumn INT, abc_id INT, " + + EMSqlScriptRunner.execCommand(getConnection(), + "CREATE TABLE Xyz(id INT Primary Key, valueColumn INT, abc_id INT, " + "CONSTRAINT fk_xyz FOREIGN KEY (abc_id) REFERENCES Abc(id) )", true); InstrumentedSutStarter starter = getInstrumentedSutStarter(); From 49b4305988c1839173562bcb9e3fd1aa0348f7b1 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 29 Sep 2025 14:04:22 +0200 Subject: [PATCH 47/50] dirty hacks for FK in SutController --- .../controller/internal/SutController.java | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java index fcf40f8780..486d02c4d0 100644 --- a/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java +++ b/client-java/controller/src/main/java/org/evomaster/client/java/controller/internal/SutController.java @@ -623,6 +623,7 @@ public final void cleanAccessedTables(){ //FIXME tableDataToInit = names.stream() .filter(a-> tableInitSqlMap.keySet().stream().anyMatch(t-> isSameTable(t, a))) + .map(String::toLowerCase) .collect(Collectors.toSet()); } } @@ -652,9 +653,31 @@ private String wrapWithQuotationMarks(String wrap, String mark) { private void handleInitSqlInDbClean(Collection tableDataToInit, DbSpecification spec) throws SQLException { // init db script //boolean initAll = registerInitSqlCommands(getConnectionIfExist(), spec); - if (tableDataToInit!= null &&!tableDataToInit.isEmpty()){ - tableDataToInit.stream().sorted((s1, s2)-> tableFkCompartor(s1, s2)).forEach(a->{ - tableInitSqlMap.keySet().stream().filter(t-> isSameTable(t, a)).forEach(t->{ + if (tableDataToInit == null || tableDataToInit.isEmpty()) { + return; + } + + tableDataToInit.stream() + .sorted((s1, s2)-> tableFkCompartor(s1, s2)) + .forEach(a->{ + tableInitSqlMap.keySet().stream() + .filter(t-> isSameTable(t, a)) + .forEach(t->{ + tableInitSqlMap.get(t).forEach(c->{ + try { + SqlScriptRunner.execCommand(getConnectionIfExist(), c); + } catch (SQLException e) { + throw new RuntimeException("SQL Init Execution Error: fail to execute "+ c + " with error "+e); + } + }); + }); + }); + } + + private void reAddAllInitSql() throws SQLException{ + + tableInitSqlMap.keySet() + .forEach(t->{ tableInitSqlMap.get(t).forEach(c->{ try { SqlScriptRunner.execCommand(getConnectionIfExist(), c); @@ -663,22 +686,6 @@ private void handleInitSqlInDbClean(Collection tableDataToInit, DbSpecif } }); }); - }); - } - } - - private void reAddAllInitSql() throws SQLException{ - if(tableInitSqlMap != null){ - tableInitSqlMap.keySet().stream().forEach(t->{ - tableInitSqlMap.get(t).forEach(c->{ - try { - SqlScriptRunner.execCommand(getConnectionIfExist(), c); - } catch (SQLException e) { - throw new RuntimeException("SQL Init Execution Error: fail to execute "+ c + " with error "+e); - } - }); - }); - } } private int tableFkCompartor(String tableA, String tableB){ @@ -686,19 +693,34 @@ private int tableFkCompartor(String tableA, String tableB){ } private int getFkDepth(String tableName, Set checked){ - List keys = fkMap.keySet().stream().filter(s-> s.equalsIgnoreCase(tableName)).collect(Collectors.toList()); - if(keys.isEmpty()) return -1; + + List keys = fkMap.keySet().stream() + //.filter(s-> s.equalsIgnoreCase(tableName)) + //FIXME + .filter((s-> s.toLowerCase().endsWith(tableName.toLowerCase()))) + .collect(Collectors.toList()); + + if(keys.isEmpty()){ + return -1; + } + checked.add(tableName); - List fks = keys.stream().map(s -> fkMap.get(s)).flatMap(List::stream).collect(Collectors.toList()); + + List fks = keys.stream() + .map(s -> fkMap.get(s)) + .flatMap(List::stream) + .collect(Collectors.toList()); if (fks.isEmpty()) { return 0; } + int sum = fks.size(); for (String fk: fks){ if (!checked.stream().anyMatch(c -> c.equalsIgnoreCase(fk))){ sum += getFkDepth(fk, checked); } } + return sum; } From ba83c2793e733677680add7185cf4a305afa5332 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 7 Oct 2025 12:05:22 +0200 Subject: [PATCH 48/50] fix for some failing tests --- .../db/directintwithsql/DbDirectIntWithSqlEMTest.java | 2 +- .../db/directintwithsql/DbDirectIntWithSqlEMTest.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index 8a87d78f0a..08bc6b29d4 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -143,7 +143,7 @@ public void testSteps(boolean heuristicsForSQLAdvanced) { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,"public"), Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call diff --git a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java index aa6b7f546f..e54dd0c9e3 100644 --- a/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java +++ b/e2e-tests/spring-rest-openapi-v2/src/test/java/org/evomaster/e2etests/spring/examples/db/directintwithsql/DbDirectIntWithSqlEMTest.java @@ -144,7 +144,9 @@ public void testSteps(boolean heuristicsForSQLAdvanced) { //now, try to execute an action in which as well we add SQL data - List insertions = sampler.sampleSqlInsertion(new TableId("DB_DIRECT_INT_ENTITY",null,null,null), Collections.singleton("*")); + List insertions = sampler.sampleSqlInsertion( + new TableId("DB_DIRECT_INT_ENTITY",null,null,"public"), + Collections.singleton("*")); assertEquals(1, insertions.size()); //extract the x/y values from the random call From 8c34566d21659b8b74b107a675404e274cec3978 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Tue, 7 Oct 2025 13:21:42 +0200 Subject: [PATCH 49/50] actually disabling tests that were supposed to be disabled --- .../e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt index 7ab653833f..f2c349a67c 100644 --- a/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt +++ b/e2e-tests/spring-rest-multidb/src/test/kotlin/org/evomaster/e2etests/spring/multidb/MultiDbParameterizedE2ETemplate.kt @@ -38,6 +38,8 @@ abstract class MultiDbParameterizedE2ETemplate : RestTestBase(){ } } + //FIXME put back once fixed + @Disabled @ParameterizedTest @EnumSource(names = ["MYSQL","POSTGRES","H2"]) fun testRunEM(databaseType: DatabaseType) { From f050be038a59c920af3e0fa53e2bf2dd071f30e1 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 8 Oct 2025 10:37:17 +0200 Subject: [PATCH 50/50] fix for failing tests --- .../evomaster/e2etests/utils/EnterpriseTestBase.java | 12 +++++++----- .../ResourceDbMIOAndHypermutationBasicTest.java | 9 ++++++++- .../e2etests/spring/rest/basic/BasicEMTest.kt | 4 ++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 5d011a2303..2cf5b4fac0 100644 --- a/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -91,12 +91,14 @@ being not instrumented when tests start (as controllers might load them) @AfterAll public static void tearDown() { - assertTimeoutPreemptively(Duration.ofMinutes(2), () -> { - boolean stopped = remoteController.stopSUT(); - stopped = embeddedStarter.stop() && stopped; + if(remoteController != null) { + assertTimeoutPreemptively(Duration.ofMinutes(2), () -> { + boolean stopped = remoteController.stopSUT(); + stopped = embeddedStarter.stop() && stopped; - assertTrue(stopped); - }); + assertTrue(stopped); + }); + } SimpleLogger.setThreshold(SimpleLogger.Level.INFO); } diff --git a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/hypermutation/ResourceDbMIOAndHypermutationBasicTest.java b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/hypermutation/ResourceDbMIOAndHypermutationBasicTest.java index f60fe30aa2..d1a9557f3f 100644 --- a/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/hypermutation/ResourceDbMIOAndHypermutationBasicTest.java +++ b/e2e-tests/spring-rest-h2-v1/src/test/java/org/evomaster/e2etests/spring/examples/resource/hypermutation/ResourceDbMIOAndHypermutationBasicTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -103,7 +104,13 @@ public void testResourceDBHypermutation() { RestResourceNode node = rmanger.getResourceNodeFromCluster(key); assertEquals(keysToTemplate.get(key), node.getTemplates().keySet(), key); // check derived tables - assertEquals(keysToTable.get(key), node.getResourceToTable().getDerivedMap().keySet(), key); + assertEquals( + keysToTable.get(key), + node.getResourceToTable().getDerivedMap().keySet().stream() + .map(it -> it.getName()) + .collect(Collectors.toSet()), + key + ); } List calls = new ArrayList<>(); diff --git a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt index 6561a239ff..4a7f8e6bd7 100644 --- a/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt +++ b/e2e-tests/spring-rest-mysql/src/test/kotlin/org/evomaster/e2etests/spring/rest/basic/BasicEMTest.kt @@ -95,7 +95,7 @@ class BasicEMTest : RestTestBase() { val injector = init(args) val sampler = injector.getInstance(RestSampler::class.java) - val dbactions = sampler.sampleSqlInsertion(TableId("X"), setOf("*")) + val dbactions = sampler.sampleSqlInsertion(TableId("X", openGroupName = "test"), setOf("*")) assertEquals(1, dbactions.size) @@ -113,4 +113,4 @@ class BasicEMTest : RestTestBase() { } } -} \ No newline at end of file +}