From 3147fa5c500f7eafad721ba38eac8a309c10231e Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Tue, 16 Jun 2026 03:30:53 +0000 Subject: [PATCH 1/2] [SPARK-55707][SQL][TESTS] Re-enable DB2 JDBC tests by upgrading jcc to 12.1.4.0 ### What changes were proposed in this pull request? SPARK-55706 disabled the DB2 JDBC driver tests because `com.ibm.db2:jcc` 11.5.x bundles an old, unshaded copy of lz4-java (`net.jpountz.*`) that predates `LZ4BlockInputStream.newBuilder()`. On the shared test classpath it shadows the lz4-java Spark ships and breaks the builder API adopted in SPARK-54571. The jcc 12.1.x driver bundles a newer lz4-java that already provides the full `newBuilder()` builder API (including the `LZ4SafeDecompressor` overload), so the conflict no longer occurs. This: - bumps `db2.jcc.version` 11.5.9.0 -> 12.1.4.0, and - reverts the test-disabling changes from SPARK-55706. ### Why are the changes needed? To restore the DB2 JDBC test coverage disabled in SPARK-55706. ### Does this PR introduce _any_ user-facing change? No. `com.ibm.db2:jcc` is a test-only dependency. ### How was this patch tested? The DB2 integration / connection-provider suites disabled by SPARK-55706 are re-enabled. The 12.1.x driver is backward-compatible with the DB2 11 test container. --- connector/docker-integration-tests/pom.xml | 5 ++--- .../spark/sql/jdbc/DB2IntegrationSuite.scala | 5 ++--- .../sql/jdbc/DB2KrbIntegrationSuite.scala | 2 -- .../sql/jdbc/v2/DB2IntegrationSuite.scala | 5 ++--- .../spark/sql/jdbc/v2/DB2NamespaceSuite.scala | 6 +----- pom.xml | 7 +++---- sql/core/pom.xml | 5 ++--- .../connection/ConnectionProviderSuite.scala | 20 +++++++++---------- .../DB2ConnectionProviderSuite.scala | 3 --- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 10 +++------- 10 files changed, 25 insertions(+), 43 deletions(-) diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml index f83c6f8fbd217..c297bd0ab6dc5 100644 --- a/connector/docker-integration-tests/pom.xml +++ b/connector/docker-integration-tests/pom.xml @@ -100,12 +100,11 @@ ojdbc17 test - - + com.microsoft.sqlserver mssql-jdbc diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala index 39aef6d7c90b3..6a489ffb2d42e 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala @@ -21,13 +21,12 @@ import java.math.BigDecimal import java.sql.{Connection, Date, Timestamp} import java.util.Properties -import org.scalatest.Ignore - import org.apache.spark.sql.{Row, SaveMode} import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ByteType, ShortType, StructType} +import org.apache.spark.tags.DockerTest /** * To run this test suite for a specific version (e.g., icr.io/db2_community/db2:11.5.9.0): @@ -37,7 +36,7 @@ import org.apache.spark.sql.types.{ByteType, ShortType, StructType} * "docker-integration-tests/testOnly org.apache.spark.sql.jdbc.DB2IntegrationSuite" * }}} */ -@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests +@DockerTest class DB2IntegrationSuite extends SharedJDBCIntegrationSuite { override val db = new DB2DatabaseOnDocker diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala index a34c1f5590a0d..c7d8fc43393ed 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2KrbIntegrationSuite.scala @@ -24,7 +24,6 @@ import javax.security.auth.login.Configuration import com.github.dockerjava.api.model.{AccessMode, Bind, ContainerConfig, HostConfig, Volume} import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation} import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS -import org.scalatest.Ignore import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.execution.datasources.jdbc.connection.{DB2ConnectionProvider, SecureConnectionProvider} @@ -38,7 +37,6 @@ import org.apache.spark.tags.DockerTest * "docker-integration-tests/testOnly *DB2KrbIntegrationSuite" * }}} */ -@Ignore // TODO(SPARK-55707: Re-enable DB2 JDBC Driver tests) @DockerTest class DB2KrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite { override protected val userName = s"db2/$dockerIp" diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala index e0b937a61fecb..ecc02f7051787 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala @@ -20,13 +20,12 @@ package org.apache.spark.sql.jdbc.v2 import java.sql.Connection import java.util.Locale -import org.scalatest.Ignore - import org.apache.spark.SparkConf import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog import org.apache.spark.sql.jdbc.DB2DatabaseOnDocker import org.apache.spark.sql.types._ +import org.apache.spark.tags.DockerTest /** * To run this test suite for a specific version (e.g., icr.io/db2_community/db2:11.5.9.0): @@ -35,7 +34,7 @@ import org.apache.spark.sql.types._ * ./build/sbt -Pdocker-integration-tests "testOnly *v2.DB2IntegrationSuite" * }}} */ -@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests +@DockerTest class DB2IntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest { // Following tests are disabled for both single and multiple partition read diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala index d96f4bf7011e5..6765652168073 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala @@ -21,8 +21,6 @@ import java.sql.Connection import scala.jdk.CollectionConverters._ -import org.scalatest.Ignore - import org.apache.spark.sql.jdbc.{DB2DatabaseOnDocker, DockerJDBCIntegrationSuite} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.tags.DockerTest @@ -34,7 +32,6 @@ import org.apache.spark.tags.DockerTest * ./build/sbt -Pdocker-integration-tests "testOnly *v2.DB2NamespaceSuite" * }}} */ -@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests @DockerTest class DB2NamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceTest { override val db = new DB2DatabaseOnDocker @@ -42,8 +39,7 @@ class DB2NamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceT Map("url" -> db.getJdbcUrl(dockerIp, externalPort), "driver" -> "com.ibm.db2.jcc.DB2Driver").asJava) - // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests - // catalog.initialize("db2", map) + catalog.initialize("db2", map) override def dataPreparation(conn: Connection): Unit = {} diff --git a/pom.xml b/pom.xml index 727c007126504..6a6bdf9c21334 100644 --- a/pom.xml +++ b/pom.xml @@ -348,7 +348,7 @@ 3.5.7 9.6.0 42.7.11 - 11.5.9.0 + 12.1.4.0 13.2.1.jre11 23.26.1.0.0 3.3.1 @@ -1382,13 +1382,12 @@ ${postgresql.version} test - - + com.microsoft.sqlserver mssql-jdbc diff --git a/sql/core/pom.xml b/sql/core/pom.xml index e6673c9069f42..84ed643b300b2 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -211,12 +211,11 @@ postgresql test - - + com.ibm.icu icu4j diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala index b2e788b52109e..0d7b133f0e15e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/ConnectionProviderSuite.scala @@ -185,28 +185,28 @@ class ConnectionProviderSuite val postgresDriver = registerDriver(postgresProvider.driverClass) val postgresOptions = options("jdbc:postgresql://localhost/postgres") val postgresAppEntry = postgresProvider.appEntry(postgresDriver, postgresOptions) - val mysqlProvider = new MariaDBConnectionProvider() - val mysqlDriver = registerDriver(mysqlProvider.driverClass) - val mysqlOptions = options("jdbc:mysql://localhost/db") - val mysqlAppEntry = mysqlProvider.appEntry(mysqlDriver, mysqlOptions) + val db2Provider = new DB2ConnectionProvider() + val db2Driver = registerDriver(db2Provider.driverClass) + val db2Options = options("jdbc:db2://localhost/db2") + val db2AppEntry = db2Provider.appEntry(db2Driver, db2Options) // Make sure no authentication for the databases are set val rootConfig = Configuration.getConfiguration assert(rootConfig.getAppConfigurationEntry(postgresAppEntry) == null) - assert(rootConfig.getAppConfigurationEntry(mysqlAppEntry) == null) + assert(rootConfig.getAppConfigurationEntry(db2AppEntry) == null) postgresProvider.setAuthenticationConfig(postgresDriver, postgresOptions) val postgresConfig = Configuration.getConfiguration - mysqlProvider.setAuthenticationConfig(mysqlDriver, mysqlOptions) - val mysqlConfig = Configuration.getConfiguration + db2Provider.setAuthenticationConfig(db2Driver, db2Options) + val db2Config = Configuration.getConfiguration // Make sure authentication for the databases are set assert(rootConfig != postgresConfig) - assert(rootConfig != mysqlConfig) + assert(rootConfig != db2Config) // The topmost config in the chain is linked with all the subsequent entries - assert(mysqlConfig.getAppConfigurationEntry(postgresAppEntry) != null) - assert(mysqlConfig.getAppConfigurationEntry(mysqlAppEntry) != null) + assert(db2Config.getAppConfigurationEntry(postgresAppEntry) != null) + assert(db2Config.getAppConfigurationEntry(db2AppEntry) != null) Configuration.setConfiguration(null) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala index f29a1b1e7ab62..895b3d85d960b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/DB2ConnectionProviderSuite.scala @@ -17,9 +17,6 @@ package org.apache.spark.sql.execution.datasources.jdbc.connection -import org.scalatest.Ignore - -@Ignore // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests class DB2ConnectionProviderSuite extends ConnectionProviderSuiteBase { test("setAuthenticationConfig must set authentication all the time") { val provider = new DB2ConnectionProvider() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 3a83a2549f924..3c04c517a670c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -1265,8 +1265,7 @@ class JDBCSuite extends SharedSparkSession { "SELECT TOP (123) a,b FROM test") } - // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests - ignore("SPARK-42534: DB2Dialect Limit query test") { + test("SPARK-42534: DB2Dialect Limit query test") { // JDBC url is a required option but is not used in this test. val options = new JDBCOptions(Map("url" -> "jdbc:db2://host:port", "dbtable" -> "test")) assert( @@ -2439,9 +2438,7 @@ class JDBCSuite extends SharedSparkSession { } // not supported Seq( - // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests - // "jdbc:db2://host:port", - "jdbc:derby:memory", "jdbc:h2://host:port", + "jdbc:db2://host:port", "jdbc:derby:memory", "jdbc:h2://host:port", "jdbc:sqlserver://host:port", "jdbc:postgresql://host:5432/postgres", "jdbc:snowflake://host:443?account=test", "jdbc:teradata://host:port").foreach { url => val options = new JDBCOptions(baseParameters + ("url" -> url)) @@ -2462,8 +2459,7 @@ class JDBCSuite extends SharedSparkSession { "jdbc:mysql", "jdbc:postgresql", "jdbc:sqlserver", - // TODO(SPARK-55707): Re-enable DB2 JDBC Driver tests - // "jdbc:db2", + "jdbc:db2", "jdbc:h2", "jdbc:teradata", "jdbc:databricks" From 918e14e6b9e3bfdd0e4d632eb4566ffd1e34277a Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Tue, 16 Jun 2026 21:06:54 +0000 Subject: [PATCH 2/2] [SPARK-55707][SQL][TESTS] Initialize DB2NamespaceSuite catalog in beforeAll Move `catalog.initialize("db2", map)` from the class body into an overridden `beforeAll()` after `super.beforeAll()`, matching the other JDBC namespace suites (Postgres/MySQL/MsSqlServer). The class-body statement ran at construction time and forced the lazy `map` while `externalPort` was still its default `-1`, producing a JDBC URL like `jdbc:db2://host:-1/foo` and failing the connection in CI. Co-authored-by: Isaac --- .../org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala index 6765652168073..52ef902d9c004 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2NamespaceSuite.scala @@ -39,7 +39,10 @@ class DB2NamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceT Map("url" -> db.getJdbcUrl(dockerIp, externalPort), "driver" -> "com.ibm.db2.jcc.DB2Driver").asJava) - catalog.initialize("db2", map) + override def beforeAll(): Unit = { + super.beforeAll() + catalog.initialize("db2", map) + } override def dataPreparation(conn: Connection): Unit = {}