From b9e7f72e9dcc79c39caa57dd3250ce43f889ecd8 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 25 Nov 2025 11:38:21 +0000 Subject: [PATCH 1/5] add config in cast/try_cast to char/varchar --- .../utils/velox/VeloxTestSettings.scala | 4 +-- .../expressions/GlutenCastSuite.scala | 20 +++++++++++++++ .../expressions/GlutenTryCastSuite.scala | 25 ++++++++++++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 9e6b8050bd5..5d7e8e38c6f 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -102,7 +102,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") - // TODO: fix in Spark-4.0 + // Set `spark.sql.preserveCharVarcharTypeInfo=true` through config. .exclude("Casting to char/varchar") enableSuite[GlutenTryCastSuite] .exclude( @@ -119,7 +119,7 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") - // TODO: fix in Spark-4.0 + // Set `spark.sql.preserveCharVarcharTypeInfo=true` through config. .exclude("Casting to char/varchar") enableSuite[GlutenCollectionExpressionsSuite] // Rewrite in Gluten to replace Seq with Array diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index ef06a985397..0c551c8d452 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -288,4 +288,24 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis)) } + testGluten("Casting to char/varchar") { + withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { + Seq(CharType(10), VarcharType(10)).foreach { typ => + Seq( + IntegerType -> (123, "123"), + LongType -> (123L, "123"), + BooleanType -> (true, "true"), + BooleanType -> (false, "false"), + DoubleType -> (1.2, "1.2") + ).foreach { case (fromType, (from, to)) => + val paddedTo = if (typ.isInstanceOf[CharType]) { + to.padTo(10, ' ') + } else { + to + } + checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) + } + } + } + } } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala index 063b6469811..d8e0379efa3 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala @@ -20,7 +20,7 @@ import org.apache.spark.sql.GlutenTestsTrait import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, TimestampType} +import org.apache.spark.sql.types._ import org.apache.spark.util.DebuggableThreadUtils import java.sql.{Date, Timestamp} @@ -214,4 +214,27 @@ class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait { } } } + + testGluten("Casting to char/varchar") { + withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { + Seq(CharType(10), VarcharType(10)).foreach { + typ => + Seq( + IntegerType -> (123, "123"), + LongType -> (123L, "123"), + BooleanType -> (true, "true"), + BooleanType -> (false, "false"), + DoubleType -> (1.2, "1.2") + ).foreach { + case (fromType, (from, to)) => + val paddedTo = if (typ.isInstanceOf[CharType]) { + to.padTo(10, ' ') + } else { + to + } + checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) + } + } + } + } } From 6535d16e8a3c5afef65d6fb47897904f3001455b Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Tue, 25 Nov 2025 13:21:40 +0000 Subject: [PATCH 2/5] fix --- .../expressions/GlutenCastSuite.scala | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index 0c551c8d452..562c5a75aae 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -290,21 +290,23 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { testGluten("Casting to char/varchar") { withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { - Seq(CharType(10), VarcharType(10)).foreach { typ => - Seq( - IntegerType -> (123, "123"), - LongType -> (123L, "123"), - BooleanType -> (true, "true"), - BooleanType -> (false, "false"), - DoubleType -> (1.2, "1.2") - ).foreach { case (fromType, (from, to)) => - val paddedTo = if (typ.isInstanceOf[CharType]) { - to.padTo(10, ' ') - } else { - to + Seq(CharType(10), VarcharType(10)).foreach { + typ => + Seq( + IntegerType -> (123, "123"), + LongType -> (123L, "123"), + BooleanType -> (true, "true"), + BooleanType -> (false, "false"), + DoubleType -> (1.2, "1.2") + ).foreach { + case (fromType, (from, to)) => + val paddedTo = if (typ.isInstanceOf[CharType]) { + to.padTo(10, ' ') + } else { + to + } + checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) } - checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) - } } } } From 6c7e4920a794e08f1588fa9dc64e267203a12092 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Nov 2025 09:22:27 +0000 Subject: [PATCH 3/5] add comments --- .../spark/sql/catalyst/expressions/GlutenCastSuite.scala | 4 ++++ .../spark/sql/catalyst/expressions/GlutenTryCastSuite.scala | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index 562c5a75aae..23f6624e707 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -289,6 +289,10 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { } testGluten("Casting to char/varchar") { + // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test + // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, + // which requires this configuration to be set. + // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { Seq(CharType(10), VarcharType(10)).foreach { typ => diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala index d8e0379efa3..0839fc0267b 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala @@ -216,6 +216,10 @@ class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait { } testGluten("Casting to char/varchar") { + // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test + // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, + // which requires this configuration to be set. + // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { Seq(CharType(10), VarcharType(10)).foreach { typ => From 9a5b5712f83206283376ddf2c87010fab4fdb8f3 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Nov 2025 11:09:44 +0000 Subject: [PATCH 4/5] address comments --- .../utils/velox/VeloxTestSettings.scala | 20 +++++------ .../expressions/GlutenCastSuite.scala | 35 +++++-------------- .../expressions/GlutenTryCastSuite.scala | 35 +++++-------------- 3 files changed, 24 insertions(+), 66 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 5d7e8e38c6f..534779e6362 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -102,8 +102,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("cast from timestamp II") .exclude("SPARK-36286: invalid string cast to timestamp") .exclude("SPARK-39749: cast Decimal to string") - // Set `spark.sql.preserveCharVarcharTypeInfo=true` through config. - .exclude("Casting to char/varchar") enableSuite[GlutenTryCastSuite] .exclude( "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. @@ -119,8 +117,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("data type casting") // Revised by setting timezone through config and commented unsupported cases. .exclude("cast string to timestamp") - // Set `spark.sql.preserveCharVarcharTypeInfo=true` through config. - .exclude("Casting to char/varchar") enableSuite[GlutenCollectionExpressionsSuite] // Rewrite in Gluten to replace Seq with Array .exclude("Shuffle") @@ -720,14 +716,14 @@ class VeloxTestSettings extends BackendTestSettings { // Extra ColumnarToRow is needed to transform vanilla columnar data to gluten columnar data. .exclude("SPARK-37369: Avoid redundant ColumnarToRow transition on InMemoryTableScan") // TODO: fix in Spark-4.0 - // enableSuite[GlutenFileSourceCharVarcharTestSuite] - // .exclude("length check for input string values: nested in array") - // .exclude("length check for input string values: nested in array") - // .exclude("length check for input string values: nested in map key") - // .exclude("length check for input string values: nested in map value") - // .exclude("length check for input string values: nested in both map key and value") - // .exclude("length check for input string values: nested in array of struct") - // .exclude("length check for input string values: nested in array of array") + enableSuite[GlutenFileSourceCharVarcharTestSuite] + .exclude("length check for input string values: nested in array") + .exclude("length check for input string values: nested in array") + .exclude("length check for input string values: nested in map key") + .exclude("length check for input string values: nested in map value") + .exclude("length check for input string values: nested in both map key and value") + .exclude("length check for input string values: nested in array of struct") + .exclude("length check for input string values: nested in array of array") // enableSuite[GlutenDSV2CharVarcharTestSuite] enableSuite[GlutenColumnExpressionSuite] // Velox raise_error('errMsg') throws a velox_user_error exception with the message 'errMsg'. diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala index 23f6624e707..c8b8aaf99a5 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala @@ -27,6 +27,14 @@ import java.sql.{Date, Timestamp} import java.util.{Calendar, TimeZone} class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { + override def beforeAll(): Unit = { + super.beforeAll() + // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test + // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, + // which requires this configuration to be set. + // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. + conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) + } override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): Cast = { v match { @@ -287,31 +295,4 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { val d = Decimal(c.getTimeInMillis.toDouble / 1000) checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis)) } - - testGluten("Casting to char/varchar") { - // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test - // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, - // which requires this configuration to be set. - // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. - withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { - Seq(CharType(10), VarcharType(10)).foreach { - typ => - Seq( - IntegerType -> (123, "123"), - LongType -> (123L, "123"), - BooleanType -> (true, "true"), - BooleanType -> (false, "false"), - DoubleType -> (1.2, "1.2") - ).foreach { - case (fromType, (from, to)) => - val paddedTo = if (typ.isInstanceOf[CharType]) { - to.padTo(10, ' ') - } else { - to - } - checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) - } - } - } - } } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala index 0839fc0267b..fc15ebfeef8 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala @@ -27,6 +27,14 @@ import java.sql.{Date, Timestamp} import java.util.{Calendar, TimeZone} class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait { + override def beforeAll(): Unit = { + super.beforeAll() + // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test + // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, + // which requires this configuration to be set. + // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. + conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) + } testGluten("data type casting") { val sd = "1970-01-01" @@ -214,31 +222,4 @@ class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait { } } } - - testGluten("Casting to char/varchar") { - // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for gluten's test - // framework. In Gluten, it overrides the checkEvaluation that invokes Spark's RowEncoder, - // which requires this configuration to be set. - // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. - withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { - Seq(CharType(10), VarcharType(10)).foreach { - typ => - Seq( - IntegerType -> (123, "123"), - LongType -> (123L, "123"), - BooleanType -> (true, "true"), - BooleanType -> (false, "false"), - DoubleType -> (1.2, "1.2") - ).foreach { - case (fromType, (from, to)) => - val paddedTo = if (typ.isInstanceOf[CharType]) { - to.padTo(10, ' ') - } else { - to - } - checkEvaluation(cast(Literal.create(from, fromType), typ), paddedTo) - } - } - } - } } From 45209df0957fc1576751732e80b215a5161cd2a5 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Wed, 26 Nov 2025 11:18:08 +0000 Subject: [PATCH 5/5] fix --- .../gluten/utils/velox/VeloxTestSettings.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 534779e6362..89f04eff45d 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -716,14 +716,14 @@ class VeloxTestSettings extends BackendTestSettings { // Extra ColumnarToRow is needed to transform vanilla columnar data to gluten columnar data. .exclude("SPARK-37369: Avoid redundant ColumnarToRow transition on InMemoryTableScan") // TODO: fix in Spark-4.0 - enableSuite[GlutenFileSourceCharVarcharTestSuite] - .exclude("length check for input string values: nested in array") - .exclude("length check for input string values: nested in array") - .exclude("length check for input string values: nested in map key") - .exclude("length check for input string values: nested in map value") - .exclude("length check for input string values: nested in both map key and value") - .exclude("length check for input string values: nested in array of struct") - .exclude("length check for input string values: nested in array of array") + // enableSuite[GlutenFileSourceCharVarcharTestSuite] + // .exclude("length check for input string values: nested in array") + // .exclude("length check for input string values: nested in array") + // .exclude("length check for input string values: nested in map key") + // .exclude("length check for input string values: nested in map value") + // .exclude("length check for input string values: nested in both map key and value") + // .exclude("length check for input string values: nested in array of struct") + // .exclude("length check for input string values: nested in array of array") // enableSuite[GlutenDSV2CharVarcharTestSuite] enableSuite[GlutenColumnExpressionSuite] // Velox raise_error('errMsg') throws a velox_user_error exception with the message 'errMsg'.