diff --git a/core/src/main/kotlin/org/evomaster/core/database/sql/solver/SmtLibGenerator.kt b/core/src/main/kotlin/org/evomaster/core/database/sql/solver/SmtLibGenerator.kt index 67e2a8eabd..12fa28cd51 100644 --- a/core/src/main/kotlin/org/evomaster/core/database/sql/solver/SmtLibGenerator.kt +++ b/core/src/main/kotlin/org/evomaster/core/database/sql/solver/SmtLibGenerator.kt @@ -113,6 +113,13 @@ class SmtLibGenerator( */ val BOOLEAN_TYPES = setOf(BOOLEAN_TYPE, "BOOL") + /** + * Spellings of a binary large object: H2 2.x reports BINARY LARGE OBJECT, H2 1.4 BLOB, and + * MySQL BLOB and its size variants. Encoded as SMT Strings; SMTLibZ3DbConstraintSolver writes + * the value in a form the database accepts. + */ + val BLOB_TYPES = setOf("BLOB", "BINARY LARGE OBJECT", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB") + /** * The canonical string values a BOOLEAN column may take (BOOLEAN is encoded as an SMT String). * These are generation constraints, so Z3 is forced to pick one of them; only the two canonical @@ -180,7 +187,7 @@ class SmtLibGenerator( JSON_TYPE to SMT_STRING, JSONB_TYPE to SMT_STRING, "BYTEA" to SMT_STRING, - ) + ) + BLOB_TYPES.associateWith { SMT_STRING } } /** diff --git a/core/src/main/kotlin/org/evomaster/core/database/sql/solver/service/SMTLibZ3DbConstraintSolver.kt b/core/src/main/kotlin/org/evomaster/core/database/sql/solver/service/SMTLibZ3DbConstraintSolver.kt index 156e13ff3a..7b25b6c106 100644 --- a/core/src/main/kotlin/org/evomaster/core/database/sql/solver/service/SMTLibZ3DbConstraintSolver.kt +++ b/core/src/main/kotlin/org/evomaster/core/database/sql/solver/service/SMTLibZ3DbConstraintSolver.kt @@ -607,7 +607,8 @@ class SMTLibZ3DbConstraintSolver() : DbConstraintSolver { * deterministically: equal strings give equal values and different strings different ones, so * the equalities and distinctions the formula imposes (foreign keys, unique and primary keys) * still hold. Constraining the form inside the formula instead would need Z3's regular - * expressions, which make solving noticeably more expensive. + * expressions, which make solving noticeably more expensive. Any other value goes on to + * [binaryValue]. */ private fun validTextValue(schemaDto: DbInfoDto, table: Table, columnName: String, value: String): String = when { @@ -616,7 +617,7 @@ class SMTLibZ3DbConstraintSolver() : DbConstraintSolver { (hasColumnType(schemaDto, table, columnName, SmtLibGenerator.JSON_TYPE) || hasColumnType(schemaDto, table, columnName, SmtLibGenerator.JSONB_TYPE)) && !isJson(value) -> JSON_MAPPER.writeValueAsString(value) // the value as a JSON string - else -> value + else -> binaryValue(schemaDto, table, columnName, value) } private fun isJson(value: String): Boolean = @@ -627,6 +628,23 @@ class SMTLibZ3DbConstraintSolver() : DbConstraintSolver { false } + /** + * Writes the value Z3 assigned to a BLOB column in a form H2 accepts. + * + * H2 1.4 reads a string literal given for a binary column as hexadecimal, so a value such as "abc" + * made it reject the INSERT of the whole row. The value is written as the hex of its UTF-8 bytes, + * which H2 2.x accepts too. The mapping is injective, so the equalities and distinctions the + * formula imposes (unique and primary keys) still hold. MySQL takes any string as is. + */ + private fun binaryValue(schemaDto: DbInfoDto, table: Table, columnName: String, value: String): String { + if (schemaDto.databaseType != DatabaseType.H2) return value + val type = schemaDto.tables.firstOrNull { it.id.name.equals(table.id.name, ignoreCase = true) } + ?.columns?.firstOrNull { it.name.equals(columnName, ignoreCase = true) } + ?.type?.uppercase() + if (type !in SmtLibGenerator.BLOB_TYPES) return value + return value.toByteArray(StandardCharsets.UTF_8).joinToString("") { "%02x".format(it) } + } + /** * Whether a column's declared type is the expected one, accounting for spellings that * [SmtLibGenerator.TYPE_MAP] already treats as equivalent. diff --git a/core/src/test/kotlin/org/evomaster/core/database/sql/solver/UnmappedColumnTypeTest.kt b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/UnmappedColumnTypeTest.kt index 5ff28aa1ae..043cd3b89b 100644 --- a/core/src/test/kotlin/org/evomaster/core/database/sql/solver/UnmappedColumnTypeTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/UnmappedColumnTypeTest.kt @@ -32,8 +32,8 @@ class UnmappedColumnTypeTest { SqlScriptRunner.execCommand( connection, "CREATE TABLE users(id bigint primary key, name varchar(255));\n" + - "CREATE TABLE blobs(id bigint primary key, payload blob);\n" + - "CREATE TABLE blob_refs(id bigint primary key, blob_id bigint REFERENCES blobs(id));\n" + "CREATE TABLE periods(id bigint primary key, length interval year);\n" + + "CREATE TABLE period_refs(id bigint primary key, period_id bigint REFERENCES periods(id));\n" ) schemaDto = DbInfoExtractor.extract(connection) } @@ -53,12 +53,12 @@ class UnmappedColumnTypeTest { val smt = generate("SELECT * FROM users WHERE name = 'x'") assertTrue(smt.contains("(get-value (users__1))")) { "expected the users row to be requested:\n$smt" } - assertFalse(smt.contains("blobs", ignoreCase = true)) { "the unmapped table must not be declared:\n$smt" } + assertFalse(smt.contains("periods", ignoreCase = true)) { "the unmapped table must not be declared:\n$smt" } } @Test fun `a query on the table with the unmapped column is rejected`() { - val e = assertThrows(RuntimeException::class.java) { generate("SELECT * FROM blobs WHERE id = 1") } + val e = assertThrows(RuntimeException::class.java) { generate("SELECT * FROM periods WHERE id = 1") } assertTrue(e.message!!.contains("Unsupported column type")) { e.message } } @@ -66,6 +66,6 @@ class UnmappedColumnTypeTest { @Test fun `a query on a table referencing it through a foreign key is rejected`() { // Its rows would reference rows of a table that is not declared, and could never be inserted - assertThrows(RuntimeException::class.java) { generate("SELECT * FROM blob_refs WHERE id = 1") } + assertThrows(RuntimeException::class.java) { generate("SELECT * FROM period_refs WHERE id = 1") } } } diff --git a/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/BlobColumnSolvingTest.kt b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/BlobColumnSolvingTest.kt new file mode 100644 index 0000000000..ed472b8878 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/BlobColumnSolvingTest.kt @@ -0,0 +1,91 @@ +package org.evomaster.core.database.sql.solver.service + +import org.evomaster.client.java.controller.api.dto.database.schema.DbInfoDto +import org.evomaster.client.java.sql.DbInfoExtractor +import org.evomaster.client.java.sql.SqlScriptRunner +import org.evomaster.core.database.sql.SqlActionTransformer +import org.evomaster.core.search.gene.string.StringGene +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.sql.Connection +import java.sql.DriverManager + +/** + * A BLOB column had no entry in the type map. Since every table of the schema is declared for every + * query, a single BLOB column made the generation fail for all the queries of the schema. + * + * Its value must also be one the database accepts: H2 1.4 reads a string literal given for a binary + * column as hexadecimal, and rejected the INSERT of a value such as "abc". + */ +class BlobColumnSolvingTest { + + companion object { + private lateinit var solver: SMTLibZ3DbConstraintSolver + private lateinit var connection: Connection + private lateinit var schemaDto: DbInfoDto + + @JvmStatic + @BeforeAll + fun setup() { + connection = DriverManager.getConnection("jdbc:h2:mem:blob_column_test", "sa", "") + SqlScriptRunner.execCommand( + connection, + "CREATE TABLE file(id bigint primary key, name varchar(20), data blob);\n" + ) + schemaDto = DbInfoExtractor.extract(connection) + solver = SMTLibZ3DbConstraintSolver() + solver.initializeExecutor() + } + + @JvmStatic + @AfterAll + fun tearDown() { + connection.close() + if (this::solver.isInitialized) { + solver.close() + } + } + + private fun dataColumn() = + schemaDto.tables.single().columns.single { it.name.equals("DATA", ignoreCase = true) } + } + + @BeforeEach + fun clean() { + SqlScriptRunner.execCommand(connection, "DELETE FROM file;\n") + } + + @Test + fun `a query on a table with a BLOB column is solved`() { + assertEquals("BINARY LARGE OBJECT", dataColumn().type.uppercase()) // as H2 2.x reports it + val query = "SELECT * FROM file WHERE name = 'report'" + + val actions = solver.solve(schemaDto, query, 1) + assertEquals(1, actions.size) + + val results = SqlScriptRunner.execInsert(connection, SqlActionTransformer.transform(actions).insertions) + assertEquals(listOf(true), results.executionResults) + assertEquals(1, SqlScriptRunner.execCommand(connection, query).seeRows().size) + } + + @Test + fun `the value of a BLOB column is written in hexadecimal`() { + val reported = dataColumn().type + dataColumn().type = "BLOB" // as H2 1.4 reports it + try { + val actions = solver.solve(schemaDto, "SELECT * FROM file WHERE data = 'abc'", 1) + assertEquals(1, actions.size) + + val data = actions.single().seeTopGenes().first { it.name.equals("DATA", ignoreCase = true) } + assertEquals("616263", (data as StringGene).value) + + val results = SqlScriptRunner.execInsert(connection, SqlActionTransformer.transform(actions).insertions) + assertEquals(listOf(true), results.executionResults) + } finally { + dataColumn().type = reported + } + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/UntranslatableQueryCacheTest.kt b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/UntranslatableQueryCacheTest.kt index 267ae5f264..93c1d7ab64 100644 --- a/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/UntranslatableQueryCacheTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/database/sql/solver/service/UntranslatableQueryCacheTest.kt @@ -52,7 +52,7 @@ class UntranslatableQueryCacheTest { private const val CACHE_BOUND = 128 /** Valid SQL, run against [schemaWithUnsupportedType] so it fails while generating SMT-LIB. */ - private const val UNSUPPORTED_TYPE = "SELECT * FROM blobs WHERE id = 1" + private const val UNSUPPORTED_TYPE = "SELECT * FROM periods WHERE id = 1" @JvmStatic @BeforeAll @@ -68,7 +68,7 @@ class UntranslatableQueryCacheTest { DriverManager.getConnection("jdbc:h2:mem:untranslatable_unsupported", "sa", "") SqlScriptRunner.execCommand( unsupportedConnection, - "CREATE TABLE blobs(id bigint primary key, payload blob);\n" + "CREATE TABLE periods(id bigint primary key, length interval year);\n" ) schemaWithUnsupportedType = DbInfoExtractor.extract(unsupportedConnection) // No injector here, so the caches have to be allocated explicitly. The Z3 executor is