diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/api/src/main/scala/org/apache/spark/sql/Encoders.scala index 72cd1190ba40c..7c97d223a2a84 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Encoders.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Encoders.scala @@ -86,14 +86,14 @@ object Encoders { * * @since 4.0.0 */ - def CHAR(length: Int): Encoder[java.lang.String] = CharEncoder(length) + def CHAR(length: Int): Encoder[java.lang.String] = CharEncoder(CharType(length)) /** * An encoder for nullable varchar type. * * @since 4.0.0 */ - def VARCHAR(length: Int): Encoder[java.lang.String] = VarcharEncoder(length) + def VARCHAR(length: Int): Encoder[java.lang.String] = VarcharEncoder(VarcharType(length)) /** * An encoder for nullable string type. diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/AgnosticEncoder.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/AgnosticEncoder.scala index 57c15de4f0db4..56dc001b227d5 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/AgnosticEncoder.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/AgnosticEncoder.scala @@ -237,8 +237,10 @@ object AgnosticEncoders { // Nullable leaf encoders case object NullEncoder extends LeafEncoder[java.lang.Void](NullType) case object StringEncoder extends LeafEncoder[String](StringType) - case class CharEncoder(length: Int) extends LeafEncoder[String](CharType(length)) - case class VarcharEncoder(length: Int) extends LeafEncoder[String](VarcharType(length)) + // Carry the full constrained type (length + collation), matching GeographyEncoder / + // GeometryEncoder. Reconstructing from length alone would drop a declared collation. + case class CharEncoder(dt: CharType) extends LeafEncoder[String](dt) + case class VarcharEncoder(dt: VarcharType) extends LeafEncoder[String](dt) case object BinaryEncoder extends LeafEncoder[Array[Byte]](BinaryType) case object ScalaBigIntEncoder extends LeafEncoder[BigInt](DecimalType.BigIntDecimal) case object JavaBigIntEncoder extends LeafEncoder[JBigInt](DecimalType.BigIntDecimal) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala index 5fce0d1491ba9..51157b5344905 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala @@ -91,10 +91,10 @@ object RowEncoder extends DataTypeErrorsBase { case DoubleType => BoxedDoubleEncoder case dt: DecimalType => JavaDecimalEncoder(dt, lenientSerialization = true) case BinaryType => BinaryEncoder - case c: CharType if SqlApiConf.get.preserveCharVarcharTypeInfo => - CharEncoder(c.length) - case v: VarcharType if SqlApiConf.get.preserveCharVarcharTypeInfo => - VarcharEncoder(v.length) + case c: CharType if SqlApiConf.get.charVarcharFirstClassTypes => + CharEncoder(c) + case v: VarcharType if SqlApiConf.get.charVarcharFirstClassTypes => + VarcharEncoder(v) case s: StringType if StringHelper.isPlainString(s) => StringEncoder case TimestampType if SqlApiConf.get.datetimeJava8ApiEnabled => InstantEncoder(lenient) case TimestampType => TimestampEncoder(lenient) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala index 2e7806294ec0b..1e35890206712 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala @@ -31,11 +31,13 @@ trait SparkCharVarcharUtils { } /** - * Validate the given [[DataType]] to fail if it is char or varchar types or contains nested - * ones + * Fail if the type contains CHAR/VARCHAR unless legacy-as-string or first-class CHAR/VARCHAR is + * enabled (standard semantics or preserveCharVarcharTypeInfo). */ def failIfHasCharVarchar(dt: DataType): DataType = { - if (!SqlApiConf.get.charVarcharAsString && hasCharVarchar(dt)) { + if (SqlApiConf.get.charVarcharFirstClassTypes) { + dt + } else if (!SqlApiConf.get.charVarcharAsString && hasCharVarchar(dt)) { throw DataTypeErrors.charOrVarcharTypeAsStringUnsupportedError() } else { replaceCharVarcharWithString(dt) @@ -54,8 +56,8 @@ trait SparkCharVarcharUtils { StructType(fields.map { field => field.copy(dataType = replaceCharVarcharWithString(field.dataType)) }) - case c: CharType if !SqlApiConf.get.preserveCharVarcharTypeInfo => c.toStringType - case v: VarcharType if !SqlApiConf.get.preserveCharVarcharTypeInfo => v.toStringType + case c: CharType if !SqlApiConf.get.charVarcharFirstClassTypes => c.toStringType + case v: VarcharType if !SqlApiConf.get.charVarcharFirstClassTypes => v.toStringType case _ => dt } } diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala index 7538580fb234a..854545088c8b8 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala @@ -42,6 +42,14 @@ private[sql] trait SqlApiConf { def allowNegativeScaleOfDecimalEnabled: Boolean def charVarcharAsString: Boolean def preserveCharVarcharTypeInfo: Boolean + def charVarcharStandardSemantics: Boolean + + /** + * True when CHAR/VARCHAR may appear as first-class types in schemas and plans (either the + * legacy preserve path or SQL standard semantics). + */ + def charVarcharFirstClassTypes: Boolean = + preserveCharVarcharTypeInfo || charVarcharStandardSemantics def datetimeJava8ApiEnabled: Boolean def sessionLocalTimeZone: String def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value @@ -104,6 +112,7 @@ private[sql] object DefaultSqlApiConf extends SqlApiConf { override def allowNegativeScaleOfDecimalEnabled: Boolean = false override def charVarcharAsString: Boolean = false override def preserveCharVarcharTypeInfo: Boolean = false + override def charVarcharStandardSemantics: Boolean = false override def datetimeJava8ApiEnabled: Boolean = false override def sessionLocalTimeZone: String = TimeZone.getDefault.getID override def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value = LegacyBehaviorPolicy.CORRECTED diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala index 34467c258d6c5..9de50cd758e8a 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala @@ -159,25 +159,76 @@ case object StringHelper extends PartialOrdering[StringConstraint] { def isPlainString(s: StringType): Boolean = s.constraint == NoConstraint + /** + * Strip CHAR/VARCHAR length constraints, preserving collation. + * + * Used by transforming string expressions (upper, substr, concat, ...) so their result type is + * plain STRING even when inputs are CharType/VarcharType (SQL standard CHAR/VARCHAR R1), when + * standard semantics are on. + */ + def plainStringType(dt: DataType): DataType = dt match { + case c: CharType => c.toStringType + case v: VarcharType => v.toStringType + case other => other + } + + def plainStringType(s: StringType): StringType = s match { + case c: CharType => c.toStringType + case v: VarcharType => v.toStringType + case other => other + } + + /** + * Result type for transforming string expressions. Under + * spark.sql.charVarchar.standardSemantics.enabled, always plain STRING (R1). Under + * preserveCharVarcharTypeInfo alone, keep child type (legacy leaky path). + */ + def transformingStringResultType(dt: DataType): DataType = { + if (SqlApiConf.get.charVarcharStandardSemantics) { + plainStringType(dt) + } else { + dt + } + } + def isMoreConstrained(a: StringType, b: StringType): Boolean = gteq(a.constraint, b.constraint) + /** + * Least common string type: CHAR -> VARCHAR -> STRING, with length max(n, m) when the result + * remains CHAR or VARCHAR. + * + * When first-class CHAR/VARCHAR are off, always widens to unbounded STRING (legacy + * annotated-STRING path where Char/Varchar do not appear in plans). + */ def tightestCommonString(s1: StringType, s2: StringType): Option[StringType] = { if (s1.collationId != s2.collationId) { return None } - if (!SqlApiConf.get.preserveCharVarcharTypeInfo) { + if (!SqlApiConf.get.charVarcharFirstClassTypes) { return Some(StringType(s1.collationId)) } + // Carry the declared collation onto CHAR/VARCHAR results. This propagates the Option rather + // than the id: None means "not explicitly declared" and renders as char(n) instead of + // char(n) collate UTF8_BINARY, so an all-default LCT keeps printing (and comparing) as before. + // The two collation ids are already known to be equal here. + val collation = declaredCollation(s1).orElse(declaredCollation(s2)) Some((s1.constraint, s2.constraint) match { - case (FixedLength(l1), FixedLength(l2)) => CharType(l1.max(l2)) - case (MaxLength(l1), FixedLength(l2)) => VarcharType(l1.max(l2)) - case (FixedLength(l1), MaxLength(l2)) => VarcharType(l1.max(l2)) - case (MaxLength(l1), MaxLength(l2)) => VarcharType(l1.max(l2)) + case (FixedLength(l1), FixedLength(l2)) => new CharType(l1.max(l2), collation) + case (MaxLength(l1), FixedLength(l2)) => new VarcharType(l1.max(l2), collation) + case (FixedLength(l1), MaxLength(l2)) => new VarcharType(l1.max(l2), collation) + case (MaxLength(l1), MaxLength(l2)) => new VarcharType(l1.max(l2), collation) case _ => StringType(s1.collationId) }) } + /** The explicitly declared collation of a CHAR/VARCHAR type, if any. */ + private def declaredCollation(s: StringType): Option[Int] = s match { + case c: CharType => c.collation + case v: VarcharType => v.collation + case _ => None + } + def removeCollation(s: StringType): StringType = s match { case c: CharType => CharType(c.length) case v: VarcharType => VarcharType(v.length) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java index 1e183b9be5977..659833851ccdf 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/util/CharVarcharCodegenUtils.java @@ -64,4 +64,27 @@ public static UTF8String readSidePadding(UTF8String inputStr, int limit) { return inputStr; } } + + /** + * Read-side CHAR check under standard semantics: pad to limit, or trim trailing + * spaces then error if still longer than limit. + * + * Standard semantics require a read to observe the same value a write would have + * produced, so this is deliberately the write-side check rather than + * {@link #readSidePadding}, which tolerates over-long values. Keep the two sides + * identical: a fix to one is a fix to both. + */ + public static UTF8String charTypeReadSideCheck(UTF8String inputStr, int limit) { + return charTypeWriteSideCheck(inputStr, limit); + } + + /** + * Read-side VARCHAR check under standard semantics: allow up to limit characters, + * or trim trailing spaces then error if still longer than limit. + * + * Identical to the write-side check by design; see {@link #varcharTypeWriteSideCheck}. + */ + public static UTF8String varcharTypeReadSideCheck(UTF8String inputStr, int limit) { + return varcharTypeWriteSideCheck(inputStr, limit); + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala index b08804dccc673..d4c37f7eb5fa9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala @@ -327,10 +327,10 @@ object DeserializerBuildHelper { createDeserializerForGeographyType(path, g.dt) case g: GeometryEncoder => createDeserializerForGeometryType(path, g.dt) - case CharEncoder(length) => - createDeserializerForChar(path, returnNullable = false, length) - case VarcharEncoder(length) => - createDeserializerForVarchar(path, returnNullable = false, length) + case CharEncoder(dt) => + createDeserializerForChar(path, returnNullable = false, dt.length) + case VarcharEncoder(dt) => + createDeserializerForVarchar(path, returnNullable = false, dt.length) case StringEncoder => createDeserializerForString(path, returnNullable = false) case _: ScalaDecimalEncoder => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala index 3531ebc77c98c..370749b1d9bf9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala @@ -358,8 +358,8 @@ object SerializerBuildHelper { messageParameters = scala.collection.immutable.Map.empty) case g: GeographyEncoder => createSerializerForGeographyType(input, g.dt) case g: GeometryEncoder => createSerializerForGeometryType(input, g.dt) - case CharEncoder(length) => createSerializerForChar(input, length) - case VarcharEncoder(length) => createSerializerForVarchar(input, length) + case CharEncoder(dt) => createSerializerForChar(input, dt.length) + case VarcharEncoder(dt) => createSerializerForVarchar(input, dt.length) case StringEncoder => createSerializerForString(input) case ScalaDecimalEncoder(dt) => createSerializerForBigDecimal(input, dt) case JavaDecimalEncoder(dt, false) => createSerializerForBigDecimal(input, dt) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala index 23c416dd4b383..efac72d250ce7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala @@ -168,6 +168,10 @@ object AnsiTypeCoercion extends TypeCoercionBase { private def implicitCast( inType: DataType, expectedType: AbstractDataType): Option[DataType] = { + // R1 CHAR/VARCHAR promotion is checked first: the acceptsType case below would otherwise + // accept the constrained type unchanged, since CharType and VarcharType extend StringType. + charVarcharToPlainString(inType, expectedType).foreach(dt => return Some(dt)) + (inType, expectedType) match { // If the expected type equals the input type, no need to cast. case _ if expectedType.acceptsType(inType) => Some(inType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 6687c163efadb..36eeedaf30ed1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -363,11 +363,13 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString plan.foreachUp { case p if p.analyzed => // Skip already analyzed sub-plans - case leaf: LeafNode if !SQLConf.get.preserveCharVarcharTypeInfo && - leaf.output.map(_.dataType).exists(CharVarcharUtils.hasCharVarchar) => + case leaf: LeafNode + if !SQLConf.get.charVarcharFirstClassTypes && + leaf.output.exists(attr => CharVarcharUtils.hasCharVarchar(attr.dataType)) => throw SparkException.internalError( s"Logical plan should not have output of char/varchar type when " + - s"${SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key} is false: " + leaf) + s"${SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key} and " + + s"${SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key} are both false: " + leaf) case u: UnresolvedNamespace => u.schemaNotFound(u.multipartIdentifier) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala index 185a5503b1107..aa119b4d5b1e1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala @@ -143,7 +143,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager) allowTemp: Boolean, columns: Seq[ColumnDefinition]): ResolvedIdentifier = { val columnOutput = columns.map { col => - val dataType = if (conf.preserveCharVarcharTypeInfo) { + val dataType = if (conf.charVarcharFirstClassTypes) { col.dataType } else { replaceCharVarcharWithString(col.dataType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 7c76e692bb5ba..b6a7e8aed67b0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -202,6 +202,10 @@ object TypeCoercion extends TypeCoercionBase { private def implicitCast(inType: DataType, expectedType: AbstractDataType): Option[DataType] = { // Note that ret is nullable to avoid typing a lot of Some(...) in this local scope. // We wrap immediately an Option after this. + // R1 CHAR/VARCHAR promotion is checked first: the acceptsType case below would otherwise + // accept the constrained type unchanged, since CharType and VarcharType extend StringType. + charVarcharToPlainString(inType, expectedType).foreach(dt => return Some(dt)) + @Nullable val ret: DataType = (inType, expectedType) match { // If the expected type is already a parent of the input type, no need to cast. case _ if expectedType.acceptsType(inType) => inType diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala index 7c83a52829100..1be2d52e8842b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala @@ -51,10 +51,54 @@ import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper import org.apache.spark.sql.connector.catalog.procedures.BoundProcedure import org.apache.spark.sql.errors.DataTypeErrors.cannotMergeIncompatibleDataTypesError -import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.types.AbstractStringType +import org.apache.spark.sql.types.{ + AbstractDataType, + DataType, + StringHelper, + StringType, + TypeCollection +} abstract class TypeCoercionBase extends TypeCoercionHelper { + /** + * R1 promotion for CHAR(n)/VARCHAR(n): where a plain string is expected, promote to STRING the + * same way SHORT promotes to INT, and return the promoted type. + * + * CharType and VarcharType extend StringType, so an expectation such as + * `StringTypeWithCollation` accepts them as-is and the implicit cast rules leave the length + * constraint in place. Expressions that then require all their string inputs to share a single + * type (`overlay`, `string_agg`, ...) cannot unify CHAR(n) with STRING, and RuntimeReplaceable + * ones (`right`) build literals from the constrained type that no longer match their other + * branches. + * + * The expectation must actually mention a string type. Promoting at an `AnyDataType` site would + * strip the length from pass-through expressions such as `max`, `lag`, and `element_at`, which + * are required to preserve CHAR/VARCHAR (R2/R3). + */ + protected def charVarcharToPlainString( + inType: DataType, + expectedType: AbstractDataType): Option[DataType] = inType match { + case st: StringType + if SQLConf.get.charVarcharStandardSemantics && !StringHelper.isPlainString(st) => + val plain = StringHelper.plainStringType(st) + if (expectsStringType(expectedType) && expectedType.acceptsType(plain)) { + Some(plain) + } else { + None + } + case _ => None + } + + private def expectsStringType(expectedType: AbstractDataType): Boolean = expectedType match { + case _: StringType => true + case _: AbstractStringType => true + case TypeCollection(types) => types.exists(expectsStringType) + case _ => false + } + /** * Type coercion rule that combines multiple type coercion rules and applies them in a single tree * traversal. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 051c1b43664c3..4565c63e19227 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -1430,7 +1430,11 @@ case class Reverse(child: Expression) BinaryType, ArrayType)) - override def dataType: DataType = child.dataType + // Reversing a string transforms its content, so a CHAR/VARCHAR input yields plain STRING (R1). + // Array and binary inputs are unaffected. ImplicitTypeCasts already promotes the string branch + // (its promotion looks inside a TypeCollection), so this covers the paths that do not go + // through implicit casting, such as an expression built directly. + override def dataType: DataType = StringHelper.transformingStringResultType(child.dataType) private def resultArrayElementNullable = dataType.asInstanceOf[ArrayType].containsNull @@ -2424,7 +2428,10 @@ case class ArrayJoin( } } - override def dataType: DataType = array.dataType.asInstanceOf[ArrayType].elementType + // The joined result concatenates every element plus delimiters, so it must not inherit the + // element's CHAR/VARCHAR length constraint (R1). + override def dataType: DataType = + StringHelper.transformingStringResultType(array.dataType.asInstanceOf[ArrayType].elementType) override def prettyName: String = "array_join" @@ -3111,7 +3118,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio if (children.isEmpty) { StringType } else { - super.dataType + StringHelper.transformingStringResultType(super.dataType) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 8629f900ac435..bff416537d4bf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -640,9 +640,14 @@ case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: E override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation) - override def dataType: DataType = MapType(first.dataType, first.dataType) + // The entries are split out of the input, so they do not carry its CHAR(n)/VARCHAR(n) length + // (R1). ExpectsInputTypes does not insert a cast, so the constraint has to be dropped here. + private lazy val entryType: DataType = + StringHelper.transformingStringResultType(first.dataType) - private lazy val mapBuilder = new ArrayBasedMapBuilder(first.dataType, first.dataType) + override def dataType: DataType = MapType(entryType, entryType) + + private lazy val mapBuilder = new ArrayBasedMapBuilder(entryType, entryType) private final lazy val collationId: Int = text.dataType.asInstanceOf[StringType].collationId diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index c912216900915..a078e2b8a551e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -286,8 +286,13 @@ case class JsonTuple(children: Seq[Expression]) }.toArray } + // The extracted fields are values from inside the JSON document, so they do not carry the + // CHAR(n)/VARCHAR(n) length of the document itself (R1). + private lazy val fieldType: DataType = + StringHelper.transformingStringResultType(children.head.dataType) + override def elementSchema: StructType = StructType(fieldExpressions.zipWithIndex.map { - case (_, idx) => StructField(s"c$idx", children.head.dataType, nullable = true) + case (_, idx) => StructField(s"c$idx", fieldType, nullable = true) }) override def prettyName: String = "json_tuple" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 546d82546d034..b0cdbcc321c1d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -171,7 +171,7 @@ object Literal { case _: DayTimeIntervalType if v.isInstanceOf[Duration] => Literal(CatalystTypeConverters.createToCatalystConverter(dataType)(v), dataType) case _: ObjectType => Literal(v, dataType) - case _: CharType | _: VarcharType if SQLConf.get.preserveCharVarcharTypeInfo => + case _: CharType | _: VarcharType if SQLConf.get.charVarcharFirstClassTypes => Literal(CatalystTypeConverters.createToCatalystConverter(dataType)(v), dataType) case _ if requiresSchemaAwareNanosConversion(dataType, v) => Literal(CatalystTypeConverters.createToCatalystConverter(dataType)(v), dataType) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala index a283eb1c8143c..dbf2331d65173 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.plans.logical.{FunctionSignature, InputParameter} import org.apache.spark.sql.errors.QueryErrorsBase import org.apache.spark.sql.internal.types.StringTypeWithCollation -import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType} +import org.apache.spark.sql.types.{AbstractDataType, DataType, StringHelper, StringType} import org.apache.spark.unsafe.types.UTF8String // scalastyle:off line.size.limit @@ -290,7 +290,8 @@ case class Mask( * Returns the [[DataType]] of the result of evaluating this expression. It is invalid to query * the dataType of an unresolved expression (i.e., when `resolved` == false). */ - override def dataType: DataType = input.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(input.dataType) /** * Returns a Seq of the children of this node. Children should not change. Immutability required diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 0cf7b7db157a7..92ce8c20d3914 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -1282,7 +1282,9 @@ case class Hex(child: Expression) Seq(TypeCollection(LongType, BinaryType, StringTypeWithCollation(supportsTrimCollation = true))) override def dataType: DataType = child.dataType match { - case st: StringType => st + // Hex expands each input character to two hex digits, so a CHAR(n)/VARCHAR(n) input must not + // carry its length constraint into the result (R1). + case st: StringType => StringHelper.transformingStringResultType(st) case _ => super.dataType } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 07d1d00ec7560..78c0eacf124c4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -617,7 +617,8 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress case class StringSplit(str: Expression, regex: Expression, limit: Expression) extends TernaryExpression with ImplicitCastInputTypes { override def nullIntolerant: Boolean = true - override def dataType: DataType = ArrayType(str.dataType, containsNull = false) + override def dataType: DataType = + ArrayType(StringHelper.transformingStringResultType(str.dataType), containsNull = false) override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeBinaryLcase, StringTypeWithCollation, IntegerType) override def first: Expression = str @@ -765,7 +766,8 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio RegExpUtils.replace(pattern, s.toString, lastReplacement, i.asInstanceOf[Int]) } - override def dataType: DataType = subject.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(subject.dataType) override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeBinaryLcase, StringTypeWithCollation, StringTypeBinaryLcase, IntegerType) @@ -940,7 +942,8 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio RegExpExtractBase.extract(getLastMatcher(s, p), r.asInstanceOf[Int], prettyName) } - override def dataType: DataType = subject.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(subject.dataType) override def prettyName: String = "regexp_extract" override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { @@ -1016,7 +1019,8 @@ case class RegExpExtractAll(subject: Expression, regexp: Expression, idx: Expres RegExpExtractBase.extractAll(getLastMatcher(s, p), r.asInstanceOf[Int], prettyName) } - override def dataType: DataType = ArrayType(subject.dataType) + override def dataType: DataType = + ArrayType(StringHelper.transformingStringResultType(subject.dataType)) override def prettyName: String = "regexp_extract_all" override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 45c9e9256aed5..c9a287799a1a6 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -99,7 +99,8 @@ case class ConcatWs(children: Seq[Expression]) Seq.fill(children.size - 1)(arrayOrStr) } - override def dataType: DataType = children.head.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(children.head.dataType) override def nullable: Boolean = children.head.nullable override def foldable: Boolean = children.forall(_.foldable) @@ -453,7 +454,8 @@ trait String2StringExpression extends ImplicitCastInputTypes { def convert(v: UTF8String): UTF8String - override def dataType: DataType = child.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(child.dataType) override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation(supportsTrimCollation = true)) override def contextIndependentFoldable: Boolean = child.contextIndependentFoldable @@ -1073,7 +1075,8 @@ case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExp CollationSupport.StringReplace.genCode(src, search, replace, collationId)) } - override def dataType: DataType = srcExpr.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(srcExpr.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeNonCSAICollation(supportsTrimCollation = true), @@ -1167,7 +1170,8 @@ case class Overlay(input: Expression, replace: Expression, pos: Expression, len: this(str, replace, pos, Literal.create(-1, IntegerType)) } - override def dataType: DataType = input.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(input.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( TypeCollection( @@ -1356,7 +1360,8 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac }) } - override def dataType: DataType = srcExpr.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(srcExpr.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeNonCSAICollation(supportsTrimCollation = true), @@ -1436,7 +1441,8 @@ trait String2TrimExpression extends Expression with ImplicitCastInputTypes { protected def direction: String override def children: Seq[Expression] = srcStr +: trimStr.toSeq - override def dataType: DataType = srcStr.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(srcStr.dataType) override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringTypeWithCollation(supportsTrimCollation = true)) @@ -1993,7 +1999,8 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: override def nullIntolerant: Boolean = true final lazy val collationId: Int = first.dataType.asInstanceOf[StringType].collationId - override def dataType: DataType = strExpr.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(strExpr.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeNonCSAICollation(supportsTrimCollation = true), @@ -2206,7 +2213,8 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression) override def second: Expression = len override def third: Expression = pad - override def dataType: DataType = str.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(str.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeWithCollation(supportsTrimCollation = true), @@ -2302,7 +2310,8 @@ case class StringRPad( override def second: Expression = len override def third: Expression = pad - override def dataType: DataType = str.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(str.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeWithCollation(supportsTrimCollation = true), @@ -2358,7 +2367,8 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC override def foldable: Boolean = children.forall(_.foldable) override def contextIndependentFoldable: Boolean = children.forall(_.contextIndependentFoldable) override def nullable: Boolean = children(0).nullable - override def dataType: DataType = children(0).dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(children(0).dataType) override def inputTypes: Seq[AbstractDataType] = StringTypeWithCollation(supportsTrimCollation = true) :: @@ -2480,7 +2490,8 @@ case class InitCap(child: Expression) override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation(supportsTrimCollation = true)) - override def dataType: DataType = child.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(child.dataType) override def nullSafeEval(string: Any): Any = { CollationSupport.InitCap.exec(string.asInstanceOf[UTF8String], collationId, useICU) @@ -2517,7 +2528,8 @@ case class StringRepeat(str: Expression, times: Expression) override def nullIntolerant: Boolean = true override def left: Expression = str override def right: Expression = times - override def dataType: DataType = str.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(str.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeWithCollation(supportsTrimCollation = true), @@ -2629,7 +2641,8 @@ case class Substring(str: Expression, pos: Expression, len: Expression) this(str, pos, Literal(Integer.MAX_VALUE)) } - override def dataType: DataType = str.dataType + override def dataType: DataType = + StringHelper.transformingStringResultType(str.dataType) override def inputTypes: Seq[AbstractDataType] = Seq( @@ -2693,12 +2706,18 @@ case class Substring(str: Expression, pos: Expression, len: Expression) case class Right(str: Expression, len: Expression) extends RuntimeReplaceable with ImplicitCastInputTypes with BinaryLike[Expression] { + // Type the literal branches after R1: Substring returns plain STRING for a CHAR(n)/VARCHAR(n) + // input, so deriving the literals from str.dataType would leave the If with branches of + // different types. CheckAnalysis does not see inside a RuntimeReplaceable's replacement, so + // that mismatch would surface later as COMPLEX_EXPRESSION_UNSUPPORTED_INPUT. + private lazy val resultType: DataType = StringHelper.transformingStringResultType(str.dataType) + override lazy val replacement: Expression = If( IsNull(str), - Literal(null, str.dataType), + Literal(null, resultType), If( LessThanOrEqual(len, Literal(0)), - Literal(UTF8String.EMPTY_UTF8, str.dataType), + Literal(UTF8String.EMPTY_UTF8, resultType), new Substring(str, UnaryMinus(len, failOnError = false)) ) ) @@ -4066,8 +4085,10 @@ case class Sentences( def this(str: Expression, language: Expression) = this(str, language, Literal("")) override def nullable: Boolean = true - override def dataType: DataType = - ArrayType(ArrayType(str.dataType, containsNull = false), containsNull = false) + override def dataType: DataType = { + val elementType = StringHelper.transformingStringResultType(str.dataType) + ArrayType(ArrayType(elementType, containsNull = false), containsNull = false) + } override def inputTypes: Seq[AbstractDataType] = Seq( StringTypeWithCollation(supportsTrimCollation = true), @@ -4102,7 +4123,8 @@ case class Sentences( case class StringSplitSQL( str: Expression, delimiter: Expression) extends BinaryExpression { - override def dataType: DataType = ArrayType(str.dataType, containsNull = false) + override def dataType: DataType = + ArrayType(StringHelper.transformingStringResultType(str.dataType), containsNull = false) final lazy val collationId: Int = left.dataType.asInstanceOf[StringType].collationId override def left: Expression = str override def right: Expression = delimiter @@ -4190,6 +4212,11 @@ case class SplitPart ( case class Empty2Null(child: Expression) extends UnaryExpression with String2StringExpression { override def convert(v: UTF8String): UTF8String = if (v.numBytes() == 0) null else v + // Not a transforming function: every non-empty value is returned unchanged, so this keeps the + // child's type rather than taking the plain-STRING result that String2StringExpression gives + // its transforming implementations (R1). + override def dataType: DataType = child.dataType + override def nullable: Boolean = true override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala index 9501986bb0c5d..1b8f681758782 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala @@ -82,9 +82,10 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils { * warning message if it has char or varchar types */ def replaceCharVarcharWithStringForCast(dt: DataType): DataType = { - if (SQLConf.get.charVarcharAsString) { + // standardSemantics takes precedence over legacy charVarcharAsString. + if (SQLConf.get.charVarcharAsString && !SQLConf.get.charVarcharStandardSemantics) { replaceCharVarcharWithString(dt) - } else if (hasCharVarchar(dt) && !SQLConf.get.preserveCharVarcharTypeInfo) { + } else if (hasCharVarchar(dt) && !SQLConf.get.charVarcharFirstClassTypes) { logWarning(log"The Spark cast operator does not support char/varchar type and simply treats" + log" them as string type. Please use string type directly to avoid confusion. Otherwise," + log" you can set ${MDC(CONFIG, SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key)} " + @@ -183,7 +184,7 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils { case c: CharType if charFuncName.isDefined => StaticInvoke( classOf[CharVarcharCodegenUtils], - if (SQLConf.get.preserveCharVarcharTypeInfo) { + if (SQLConf.get.charVarcharFirstClassTypes) { c } else { c.toStringType @@ -195,7 +196,7 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils { case v: VarcharType if varcharFuncName.isDefined => StaticInvoke( classOf[CharVarcharCodegenUtils], - if (SQLConf.get.preserveCharVarcharTypeInfo) { + if (SQLConf.get.charVarcharFirstClassTypes) { v } else { v.toStringType @@ -256,9 +257,23 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils { } def addPaddingForScan(attr: Attribute): Expression = { - getRawType(attr.metadata).map { rawType => - processStringForCharVarchar( - attr, rawType, charFuncName = Some("readSidePadding"), varcharFuncName = None) + // Driven by metadata rather than attr.dataType even when Char/Varchar are first-class types. + // The metadata is the "not yet padded" marker: ApplyCharTypePadding rebuilds the relation via + // cleanAttrMetadata, so a second application of the rule finds no raw type and leaves the plan + // alone. Keying off attr.dataType instead would re-pad an already-padded scan on every pass and + // break the Once strategy's idempotence check. + getRawType(attr.metadata).map { dt => + if (SQLConf.get.charVarcharStandardSemantics) { + // Pad CHAR and enforce length limits for CHAR/VARCHAR (trim trailing blanks first). + processStringForCharVarchar( + attr, + dt, + charFuncName = Some("charTypeReadSideCheck"), + varcharFuncName = Some("varcharTypeReadSideCheck")) + } else { + processStringForCharVarchar( + attr, dt, charFuncName = Some("readSidePadding"), varcharFuncName = None) + } }.getOrElse(attr) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index ff7c005e4a26c..b0c97f65995f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -7305,13 +7305,29 @@ object SQLConf { .createWithDefault(false) val PRESERVE_CHAR_VARCHAR_TYPE_INFO = buildConf("spark.sql.preserveCharVarcharTypeInfo") - .doc("When true, Spark does not replace CHAR/VARCHAR types the STRING type, which is the " + - "default behavior of Spark 3.0 and earlier versions. This means the length checks for " + - "CHAR/VARCHAR types is enforced and CHAR type is also properly padded.") + .doc("When true, Spark does not replace CHAR/VARCHAR with STRING in schemas and plans. " + + "This is the Spark 4.0 experimental path: types can leak through transforming string " + + "functions via child.dataType. Prefer spark.sql.charVarchar.standardSemantics.enabled " + + "for SQL standard CHAR/VARCHAR behavior (CAST/LCT/STRING-returning transforms).") .version("4.0.0") .booleanConf .createWithDefault(false) + val CHAR_VARCHAR_STANDARD_SEMANTICS = + buildConf("spark.sql.charVarchar.standardSemantics.enabled") + .doc("When true, enable SQL standard CHAR/VARCHAR semantics: first-class types in " + + "schemas and CAST targets; least-common-type for COALESCE/CASE/UNION may return " + + "CHAR/VARCHAR; transforming string functions and operators return plain STRING. " + + "This is a breaking change from the annotated-STRING default and from " + + "preserveCharVarcharTypeInfo (which keeps Char/Varchar through transforms).") + .version("4.4.0") + // PERSISTED, like ANSI mode: the flag decides the types a view body resolves to, so a view + // created under standard semantics must keep computing CHAR/VARCHAR regardless of the + // caller's session setting. + .withBindingPolicy(ConfigBindingPolicy.PERSISTED) + .booleanConf + .createWithDefault(false) + val READ_FILE_SOURCE_TABLE_CACHE_IGNORE_OPTIONS = buildConf("spark.sql.legacy.readFileSourceTableCacheIgnoreOptions") .internal() @@ -7325,7 +7341,9 @@ object SQLConf { val READ_SIDE_CHAR_PADDING = buildConf("spark.sql.readSideCharPadding") .doc("When true, Spark applies string padding when reading CHAR type columns/fields, " + "in addition to the write-side padding. This config is true by default to better enforce " + - "CHAR type semantic in cases such as external tables.") + "CHAR type semantic in cases such as external tables. When " + + s"'${CHAR_VARCHAR_STANDARD_SEMANTICS.key}' is true, this config is ignored: read-side " + + "CHAR/VARCHAR checks are always applied, and setting it to false logs a warning.") .version("3.4.0") .booleanConf .createWithDefault(true) @@ -9574,6 +9592,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def preserveCharVarcharTypeInfo: Boolean = getConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO) + def charVarcharStandardSemantics: Boolean = getConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS) + def avoidDoubleFilterEval: Boolean = getConf(AVOID_DOUBLE_FILTER_EVAL) def structPredicateDecomposeEnabled: Boolean = getConf(STRUCT_PREDICATE_DECOMPOSE_ENABLED) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala index 69f4995220fe2..b7a052d5fc4eb 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala @@ -592,38 +592,55 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest { test("do not allow serializing too long strings into char/varchar") { Seq(CharType(5), VarcharType(5)).foreach { typ => - withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { - val schema = new StructType().add("c", typ) - val encoder = ExpressionEncoder(schema).resolveAndBind() - val value = "abcdef" - checkError( - exception = intercept[SparkRuntimeException]({ - val row = toRow(encoder, Row(value)) - }), - condition = "EXCEED_LIMIT_LENGTH", - parameters = Map("limit" -> "5") - ) + Seq( + SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true", + SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true").foreach { conf => + withSQLConf(conf) { + val schema = new StructType().add("c", typ) + val encoder = ExpressionEncoder(schema).resolveAndBind() + val value = "abcdef" + checkError( + exception = intercept[SparkRuntimeException]({ + val row = toRow(encoder, Row(value)) + }), + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "5") + ) + } } } } test("do not allow deserializing too long strings into char/varchar") { Seq(CharType(5), VarcharType(5)).foreach { typ => - withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { - val fromSchema = new StructType().add("c", StringType) - val fromEncoder = ExpressionEncoder(fromSchema).resolveAndBind() - val toSchema = new StructType().add("c", typ) - val toEncoder = ExpressionEncoder(toSchema).resolveAndBind() - val value = "abcdef" - val row = toRow(fromEncoder, Row(value)) - checkError( - exception = intercept[SparkRuntimeException]({ - val value = fromRow(toEncoder, row) - }), - condition = "EXCEED_LIMIT_LENGTH", - parameters = Map("limit" -> "5") - ) + Seq( + SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true", + SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true").foreach { conf => + withSQLConf(conf) { + val fromSchema = new StructType().add("c", StringType) + val fromEncoder = ExpressionEncoder(fromSchema).resolveAndBind() + val toSchema = new StructType().add("c", typ) + val toEncoder = ExpressionEncoder(toSchema).resolveAndBind() + val value = "abcdef" + val row = toRow(fromEncoder, Row(value)) + checkError( + exception = intercept[SparkRuntimeException]({ + val value = fromRow(toEncoder, row) + }), + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "5") + ) + } } } } + + test("SPARK-58803: RowEncoder pads CHAR under standardSemantics") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val schema = new StructType().add("c", CharType(5)).add("v", VarcharType(5)) + val encoder = ExpressionEncoder(schema).resolveAndBind() + val row = toRow(encoder, Row("ab", "cd")) + assert(fromRow(encoder, row) === Row("ab ", "cd")) + } + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ApplyCharTypePadding.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ApplyCharTypePadding.scala index d952927f9d30a..a1cc576b2b236 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ApplyCharTypePadding.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ApplyCharTypePadding.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.execution.datasources +import java.util.concurrent.atomic.AtomicBoolean + +import org.apache.spark.internal.LogKeys import org.apache.spark.sql.catalyst.analysis.ApplyCharTypePaddingHelper import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -36,12 +39,28 @@ import org.apache.spark.sql.internal.SQLConf */ object ApplyCharTypePadding extends Rule[LogicalPlan] { + private val readSidePaddingOverrideWarned = new AtomicBoolean(false) + + private def warnReadSidePaddingOverride(): Unit = { + if (readSidePaddingOverrideWarned.compareAndSet(false, true)) { + logWarning(log"${MDC(LogKeys.CONFIG, SQLConf.READ_SIDE_CHAR_PADDING.key)} is disabled but " + + log"${MDC(LogKeys.CONFIG2, SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key)} is enabled; " + + log"read-side CHAR/VARCHAR checks are still applied because standard semantics require " + + log"a read to observe the value a write would have produced.") + } + } + override def apply(plan: LogicalPlan): LogicalPlan = { - if (conf.charVarcharAsString) { + // standardSemantics takes precedence over legacy charVarcharAsString. + if (conf.charVarcharAsString && !conf.charVarcharStandardSemantics) { return plan } - if (conf.readSideCharPadding) { + if (conf.charVarcharStandardSemantics && !conf.readSideCharPadding) { + warnReadSidePaddingOverride() + } + + if (conf.readSideCharPadding || conf.charVarcharStandardSemantics) { val newPlan = plan.resolveOperatorsUpWithNewOutput { case r: LogicalRelation => ApplyCharTypePaddingHelper.readSidePadding(r, () => diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar-standard-semantics.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar-standard-semantics.sql.out new file mode 100644 index 0000000000000..81fb27a17e343 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar-standard-semantics.sql.out @@ -0,0 +1,410 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT typeof(CAST('ab' AS CHAR(5))) +-- !query analysis +Project [typeof(cast(ab as char(5))) AS typeof(CAST(ab AS CHAR(5)))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(CAST('hello' AS VARCHAR(5))) +-- !query analysis +Project [typeof(cast(hello as varchar(5))) AS typeof(CAST(hello AS VARCHAR(5)))#x] ++- OneRowRelation + + +-- !query +SELECT 'X' || CAST('5' AS CHAR(5)) || 'X' +-- !query analysis +Project [concat(concat(X, cast(cast(5 as char(5)) as string)), X) AS concat(concat(X, CAST(5 AS CHAR(5))), X)#x] ++- OneRowRelation + + +-- !query +SELECT CAST('ab ' AS CHAR(2)) +-- !query analysis +Project [cast(ab as char(2)) AS CAST(ab AS CHAR(2))#x] ++- OneRowRelation + + +-- !query +SELECT CAST('abcdef' AS CHAR(2)) +-- !query analysis +Project [cast(abcdef as char(2)) AS CAST(abcdef AS CHAR(2))#x] ++- OneRowRelation + + +-- !query +SELECT CAST('abcdef' AS VARCHAR(2)) +-- !query analysis +Project [cast(abcdef as varchar(2)) AS CAST(abcdef AS VARCHAR(2))#x] ++- OneRowRelation + + +-- !query +SELECT try_cast('abcdef' AS CHAR(2)) +-- !query analysis +Project [try_cast(abcdef as char(2)) AS TRY_CAST(abcdef AS CHAR(2))#x] ++- OneRowRelation + + +-- !query +SELECT try_cast('abcdef' AS VARCHAR(2)) +-- !query analysis +Project [try_cast(abcdef as varchar(2)) AS TRY_CAST(abcdef AS VARCHAR(2))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world' AS VARCHAR(10)))) +-- !query analysis +Project [typeof(coalesce(cast(cast(hello as varchar(5)) as varchar(10)), cast(world as varchar(10)))) AS typeof(coalesce(CAST(hello AS VARCHAR(5)), CAST(world AS VARCHAR(10))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world!' AS CHAR(6)))) +-- !query analysis +Project [typeof(coalesce(cast(cast(hello as varchar(5)) as varchar(6)), cast(cast(world! as char(6)) as varchar(6)))) AS typeof(coalesce(CAST(hello AS VARCHAR(5)), CAST(world! AS CHAR(6))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), cast('world!' AS CHAR(6)))) +-- !query analysis +Project [typeof(coalesce(cast(cast(hello as char(5)) as char(6)), cast(world! as char(6)))) AS typeof(coalesce(CAST(hello AS CHAR(5)), CAST(world! AS CHAR(6))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), 'world')) +-- !query analysis +Project [typeof(coalesce(cast(cast(hello as varchar(5)) as string collate UTF8_BINARY), world)) AS typeof(coalesce(CAST(hello AS VARCHAR(5)), world))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), NULL)) +-- !query analysis +Project [typeof(coalesce(cast(hello as char(5)), cast(null as char(5)))) AS typeof(coalesce(CAST(hello AS CHAR(5)), NULL))#x] ++- OneRowRelation + + +-- !query +SELECT typeof( + CASE WHEN true THEN cast('a' AS CHAR(2)) ELSE cast('bb' AS CHAR(4)) END) +-- !query analysis +Project [typeof(CASE WHEN true THEN cast(cast(a as char(2)) as char(4)) ELSE cast(bb as char(4)) END) AS typeof(CASE WHEN true THEN CAST(a AS CHAR(2)) ELSE CAST(bb AS CHAR(4)) END)#x] ++- OneRowRelation + + +-- !query +SELECT cast('a' AS CHAR(2)) IN (cast('a ' AS CHAR(2)), cast('bbb' AS VARCHAR(3))) +-- !query analysis +Project [cast(cast(a as char(2)) as varchar(3)) IN (cast(cast(a as char(2)) as varchar(3)),cast(cast(bbb as varchar(3)) as varchar(3))) AS (CAST(a AS CHAR(2)) IN (CAST(a AS CHAR(2)), CAST(bbb AS VARCHAR(3))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(c) FROM (SELECT cast('a' AS CHAR(2)) AS c) t WHERE c IN ('a ', 'b') +-- !query analysis +Project [typeof(c#x) AS typeof(c)#x] ++- Filter cast(c#x as string collate UTF8_BINARY) IN (cast(a as string collate UTF8_BINARY),cast(b as string collate UTF8_BINARY)) + +- SubqueryAlias t + +- Project [cast(a as char(2)) AS c#x] + +- OneRowRelation + + +-- !query +SELECT typeof(upper(cast('ab' AS CHAR(2)))) +-- !query analysis +Project [typeof(upper(cast(cast(ab as char(2)) as string))) AS typeof(upper(CAST(ab AS CHAR(2))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(lower(cast('AB' AS VARCHAR(2)))) +-- !query analysis +Project [typeof(lower(cast(cast(AB as varchar(2)) as string))) AS typeof(lower(CAST(AB AS VARCHAR(2))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(cast('a' AS CHAR(1)) || cast('b' AS VARCHAR(1))) +-- !query analysis +Project [typeof(concat(cast(cast(a as char(1)) as string), cast(cast(b as varchar(1)) as string))) AS typeof(concat(CAST(a AS CHAR(1)), CAST(b AS VARCHAR(1))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(substr(cast('hello' AS VARCHAR(5)), 1, 2)) +-- !query analysis +Project [typeof(substr(cast(cast(hello as varchar(5)) as string), 1, 2)) AS typeof(substr(CAST(hello AS VARCHAR(5)), 1, 2))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(upper(coalesce(cast('a' AS CHAR(2)), cast('b' AS CHAR(4))))) +-- !query analysis +Project [typeof(upper(cast(coalesce(cast(cast(a as char(2)) as char(4)), cast(b as char(4))) as string))) AS typeof(upper(coalesce(CAST(a AS CHAR(2)), CAST(b AS CHAR(4)))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(concat(cast('a' AS CHAR(2)), cast('b' AS CHAR(3)))) +-- !query analysis +Project [typeof(concat(cast(cast(a as char(2)) as string), cast(cast(b as char(3)) as string))) AS typeof(concat(CAST(a AS CHAR(2)), CAST(b AS CHAR(3))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(trim(cast('ab ' AS CHAR(4)))) +-- !query analysis +Project [typeof(trim(cast(cast(ab as char(4)) as string), None)) AS typeof(trim(CAST(ab AS CHAR(4))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(lpad(cast('ab' AS CHAR(2)), 5, 'x')) +-- !query analysis +Project [typeof(lpad(cast(cast(ab as char(2)) as string), 5, x)) AS typeof(lpad(CAST(ab AS CHAR(2)), 5, x))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(regexp_replace(cast('ab' AS CHAR(2)), 'a', 'x')) +-- !query analysis +Project [typeof(regexp_replace(cast(cast(ab as char(2)) as string), a, x, 1)) AS typeof(regexp_replace(CAST(ab AS CHAR(2)), a, x, 1))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(regexp_extract(cast('ab' AS VARCHAR(2)), '(a)', 1)) +-- !query analysis +Project [typeof(regexp_extract(cast(cast(ab as varchar(2)) as string), (a), 1)) AS typeof(regexp_extract(CAST(ab AS VARCHAR(2)), (a), 1))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(regexp_extract_all(cast('aab' AS VARCHAR(3)), '(a)', 1)) +-- !query analysis +Project [typeof(regexp_extract_all(cast(cast(aab as varchar(3)) as string), (a), 1)) AS typeof(regexp_extract_all(CAST(aab AS VARCHAR(3)), (a), 1))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(split(cast('a,b' AS CHAR(3)), ',')) +-- !query analysis +Project [typeof(split(cast(cast(a,b as char(3)) as string), ,, -1)) AS typeof(split(CAST(a,b AS CHAR(3)), ,, -1))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(mask(cast('ab' AS CHAR(2)))) +-- !query analysis +Project [typeof(mask(cast(ab as char(2)), X, x, n, null)) AS typeof(mask(CAST(ab AS CHAR(2)), X, x, n, NULL))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1)) +-- !query analysis +Project [typeof(overlay(cast(cast(ab as char(5)) as string), x, 1, -1)) AS typeof(overlay(CAST(ab AS CHAR(5)), x, 1, -1))#x] ++- OneRowRelation + + +-- !query +SELECT concat('<', overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1), '>') +-- !query analysis +Project [concat(<, overlay(cast(cast(ab as char(5)) as string), x, 1, -1), >) AS concat(<, overlay(CAST(ab AS CHAR(5)), x, 1, -1), >)#x] ++- OneRowRelation + + +-- !query +SELECT typeof(elt(1, cast('ab' AS CHAR(5)), 'x')) +-- !query analysis +Project [typeof(elt(1, cast(cast(ab as char(5)) as string), x, true)) AS typeof(elt(1, CAST(ab AS CHAR(5)), x))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(right(cast('ab' AS CHAR(5)), 2)) +-- !query analysis +Project [typeof(right(cast(cast(ab as char(5)) as string), 2)) AS typeof(right(CAST(ab AS CHAR(5)), 2))#x] ++- OneRowRelation + + +-- !query +SELECT concat('<', right(cast('ab' AS CHAR(5)), 2), '>') +-- !query analysis +Project [concat(<, right(cast(cast(ab as char(5)) as string), 2), >) AS concat(<, right(CAST(ab AS CHAR(5)), 2), >)#x] ++- OneRowRelation + + +-- !query +SELECT typeof(left(cast('ab' AS CHAR(5)), 2)) +-- !query analysis +Project [typeof(left(cast(cast(ab as char(5)) as string), 2)) AS typeof(left(CAST(ab AS CHAR(5)), 2))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(reverse(cast('ab' AS CHAR(5)))) +-- !query analysis +Project [typeof(reverse(cast(cast(ab as char(5)) as string))) AS typeof(reverse(CAST(ab AS CHAR(5))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(hex(cast('ab' AS CHAR(5)))) +-- !query analysis +Project [typeof(hex(cast(cast(ab as char(5)) as string))) AS typeof(hex(CAST(ab AS CHAR(5))))#x] ++- OneRowRelation + + +-- !query +SELECT hex(cast('ab' AS CHAR(5))) +-- !query analysis +Project [hex(cast(cast(ab as char(5)) as string)) AS hex(CAST(ab AS CHAR(5)))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-')) +-- !query analysis +Project [typeof(array_join(array(cast(ab as char(5)), cast(cd as char(5))), -, None)) AS typeof(array_join(array(CAST(ab AS CHAR(5)), CAST(cd AS CHAR(5))), -))#x] ++- OneRowRelation + + +-- !query +SELECT concat('<', array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-'), '>') +-- !query analysis +Project [concat(<, array_join(array(cast(ab as char(5)), cast(cd as char(5))), -, None), >) AS concat(<, array_join(array(CAST(ab AS CHAR(5)), CAST(cd AS CHAR(5))), -), >)#x] ++- OneRowRelation + + +-- !query +SELECT typeof(reverse(array(1, 2))) +-- !query analysis +Project [typeof(reverse(array(1, 2))) AS typeof(reverse(array(1, 2)))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(str_to_map(cast('a:1,b:2' AS CHAR(7)))) +-- !query analysis +Project [typeof(str_to_map(cast(a:1,b:2 as char(7)), ,, :)) AS typeof(str_to_map(CAST(a:1,b:2 AS CHAR(7)), ,, :))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(c0) FROM (SELECT json_tuple(cast('{"a":"1"}' AS CHAR(9)), 'a') AS c0) +-- !query analysis +Project [typeof(c0#x) AS typeof(c0)#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [c0#x] + +- Generate json_tuple(cast({"a":"1"} as char(9)), a), false, [c0#x] + +- OneRowRelation + + +-- !query +SELECT typeof(cast('a' AS CHAR(2) COLLATE UTF8_LCASE)) +-- !query analysis +Project [typeof(cast(a as char(2) collate UTF8_LCASE)) AS typeof(CAST(a AS CHAR(2) COLLATE UTF8_LCASE))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(coalesce( + cast('a' AS CHAR(2) COLLATE UTF8_LCASE), cast('bb' AS CHAR(2) COLLATE UTF8_LCASE))) +-- !query analysis +Project [typeof(coalesce(cast(a as char(2) collate UTF8_LCASE), cast(bb as char(2) collate UTF8_LCASE))) AS typeof(coalesce(CAST(a AS CHAR(2) COLLATE UTF8_LCASE), CAST(bb AS CHAR(2) COLLATE UTF8_LCASE)))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(c) FROM ( + SELECT cast('a' AS VARCHAR(3)) AS c + UNION ALL + SELECT cast('abcd' AS VARCHAR(8)) AS c +) t LIMIT 1 +-- !query analysis +GlobalLimit 1 ++- LocalLimit 1 + +- Project [typeof(c#x) AS typeof(c)#x] + +- SubqueryAlias t + +- Union false, false + :- Project [cast(c#x as varchar(8)) AS c#x] + : +- Project [cast(a as varchar(3)) AS c#x] + : +- OneRowRelation + +- Project [cast(abcd as varchar(8)) AS c#x] + +- OneRowRelation + + +-- !query +SELECT typeof(array(cast('a' AS CHAR(2)), cast('bb' AS CHAR(3)))) +-- !query analysis +Project [typeof(array(cast(cast(a as char(2)) as char(3)), cast(bb as char(3)))) AS typeof(array(CAST(a AS CHAR(2)), CAST(bb AS CHAR(3))))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(struct(cast('a' AS CHAR(2)) AS f)) +-- !query analysis +Project [typeof(struct(f, cast(a as char(2)))) AS typeof(struct(CAST(a AS CHAR(2)) AS f))#x] ++- OneRowRelation + + +-- !query +SELECT typeof(map('k', cast('a' AS VARCHAR(2)))) +-- !query analysis +Project [typeof(map(k, cast(a as varchar(2)))) AS typeof(map(k, CAST(a AS VARCHAR(2))))#x] ++- OneRowRelation + + +-- !query +CREATE TABLE char_varchar_std (c CHAR(5), v VARCHAR(5)) USING parquet +-- !query analysis +CreateDataSourceTableCommand `spark_catalog`.`default`.`char_varchar_std`, false + + +-- !query +INSERT INTO char_varchar_std VALUES ('ab', 'ab') +-- !query analysis +InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/char_varchar_std, false, Parquet, [path=file:[not included in comparison]/{warehouse_dir}/char_varchar_std], Append, `spark_catalog`.`default`.`char_varchar_std`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/char_varchar_std), [c, v] ++- Project [static_invoke(CharVarcharCodegenUtils.charTypeWriteSideCheck(cast(col1#x as char(5)), 5)) AS c#x, static_invoke(CharVarcharCodegenUtils.varcharTypeWriteSideCheck(cast(col2#x as varchar(5)), 5)) AS v#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +SELECT typeof(c), typeof(v) FROM char_varchar_std +-- !query analysis +Project [typeof(c#x) AS typeof(c)#x, typeof(v#x) AS typeof(v)#x] ++- SubqueryAlias spark_catalog.default.char_varchar_std + +- Project [static_invoke(CharVarcharCodegenUtils.charTypeReadSideCheck(c#x, 5)) AS c#x, static_invoke(CharVarcharCodegenUtils.varcharTypeReadSideCheck(v#x, 5)) AS v#x] + +- Relation spark_catalog.default.char_varchar_std[c#x,v#x] parquet + + +-- !query +SELECT concat('[', c, ']'), concat('[', v, ']') FROM char_varchar_std +-- !query analysis +Project [concat([, cast(c#x as string), ]) AS concat([, c, ])#x, concat([, cast(v#x as string), ]) AS concat([, v, ])#x] ++- SubqueryAlias spark_catalog.default.char_varchar_std + +- Project [static_invoke(CharVarcharCodegenUtils.charTypeReadSideCheck(c#x, 5)) AS c#x, static_invoke(CharVarcharCodegenUtils.varcharTypeReadSideCheck(v#x, 5)) AS v#x] + +- Relation spark_catalog.default.char_varchar_std[c#x,v#x] parquet + + +-- !query +SELECT length(c), length(v) FROM char_varchar_std +-- !query analysis +Project [length(cast(c#x as string)) AS length(c)#x, length(cast(v#x as string)) AS length(v)#x] ++- SubqueryAlias spark_catalog.default.char_varchar_std + +- Project [static_invoke(CharVarcharCodegenUtils.charTypeReadSideCheck(c#x, 5)) AS c#x, static_invoke(CharVarcharCodegenUtils.varcharTypeReadSideCheck(v#x, 5)) AS v#x] + +- Relation spark_catalog.default.char_varchar_std[c#x,v#x] parquet + + +-- !query +DROP TABLE char_varchar_std +-- !query analysis +DropTable false, false ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.char_varchar_std diff --git a/sql/core/src/test/resources/sql-tests/inputs/charvarchar-standard-semantics.sql b/sql/core/src/test/resources/sql-tests/inputs/charvarchar-standard-semantics.sql new file mode 100644 index 0000000000000..6d591bc7e8cf8 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/charvarchar-standard-semantics.sql @@ -0,0 +1,97 @@ +--SET spark.sql.charVarchar.standardSemantics.enabled=true + +-- R3: CAST introduces CHAR/VARCHAR +SELECT typeof(CAST('ab' AS CHAR(5))); +SELECT typeof(CAST('hello' AS VARCHAR(5))); +SELECT 'X' || CAST('5' AS CHAR(5)) || 'X'; + +-- CAST length enforcement: trailing spaces are trimmed, real overflow errors +SELECT CAST('ab ' AS CHAR(2)); +SELECT CAST('abcdef' AS CHAR(2)); +SELECT CAST('abcdef' AS VARCHAR(2)); +SELECT try_cast('abcdef' AS CHAR(2)); +SELECT try_cast('abcdef' AS VARCHAR(2)); + +-- R2: least common type (COALESCE / CASE) +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world' AS VARCHAR(10)))); +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world!' AS CHAR(6)))); +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), cast('world!' AS CHAR(6)))); +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), 'world')); +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), NULL)); +SELECT typeof( + CASE WHEN true THEN cast('a' AS CHAR(2)) ELSE cast('bb' AS CHAR(4)) END); + +-- R2: least common type for IN lists +SELECT cast('a' AS CHAR(2)) IN (cast('a ' AS CHAR(2)), cast('bbb' AS VARCHAR(3))); +SELECT typeof(c) FROM (SELECT cast('a' AS CHAR(2)) AS c) t WHERE c IN ('a ', 'b'); + +-- R1: transforming functions return STRING +SELECT typeof(upper(cast('ab' AS CHAR(2)))); +SELECT typeof(lower(cast('AB' AS VARCHAR(2)))); +SELECT typeof(cast('a' AS CHAR(1)) || cast('b' AS VARCHAR(1))); +SELECT typeof(substr(cast('hello' AS VARCHAR(5)), 1, 2)); +SELECT typeof(upper(coalesce(cast('a' AS CHAR(2)), cast('b' AS CHAR(4))))); +SELECT typeof(concat(cast('a' AS CHAR(2)), cast('b' AS CHAR(3)))); +SELECT typeof(trim(cast('ab ' AS CHAR(4)))); +SELECT typeof(lpad(cast('ab' AS CHAR(2)), 5, 'x')); + +-- R1: regexp / mask / split family +SELECT typeof(regexp_replace(cast('ab' AS CHAR(2)), 'a', 'x')); +SELECT typeof(regexp_extract(cast('ab' AS VARCHAR(2)), '(a)', 1)); +SELECT typeof(regexp_extract_all(cast('aab' AS VARCHAR(3)), '(a)', 1)); +SELECT typeof(split(cast('a,b' AS CHAR(3)), ',')); +SELECT typeof(mask(cast('ab' AS CHAR(2)))); + +-- R1: CHAR/VARCHAR promote to STRING where a plain string is expected, so expressions that +-- require all their string inputs to share one type accept them alongside a STRING argument. +-- Values are wrapped in sentinels because the golden format trims trailing blanks, which would +-- otherwise hide the CHAR padding these expressions operate on. +SELECT typeof(overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1)); +SELECT concat('<', overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1), '>'); +SELECT typeof(elt(1, cast('ab' AS CHAR(5)), 'x')); +SELECT typeof(right(cast('ab' AS CHAR(5)), 2)); +SELECT concat('<', right(cast('ab' AS CHAR(5)), 2), '>'); +SELECT typeof(left(cast('ab' AS CHAR(5)), 2)); + +-- R1: transforms whose result length differs from the input must not inherit the constraint. +SELECT typeof(reverse(cast('ab' AS CHAR(5)))); +SELECT typeof(hex(cast('ab' AS CHAR(5)))); +SELECT hex(cast('ab' AS CHAR(5))); +SELECT typeof(array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-')); +SELECT concat('<', array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-'), '>'); +-- reverse() on a non-string input is unaffected. +SELECT typeof(reverse(array(1, 2))); +-- Expressions that pull values out of a wrapper do not inherit the wrapper's constraint. These +-- take their inputs through ExpectsInputTypes, so no implicit cast strips the length for them. +SELECT typeof(str_to_map(cast('a:1,b:2' AS CHAR(7)))); +SELECT typeof(c0) FROM (SELECT json_tuple(cast('{"a":"1"}' AS CHAR(9)), 'a') AS c0); + +-- R2 with collation. A declared collation survives the CAST and an LCT over equally constrained +-- operands. The mixed-length case (CHAR(2) with CHAR(4), same collation) is deliberately not +-- covered here: CollationTypeCoercion reads the differing lengths as a collation mismatch and +-- yields an indeterminate collation. That predates this change (it reproduces under +-- spark.sql.preserveCharVarcharTypeInfo) and is tracked separately, so goldening it would +-- normalize the bug. +SELECT typeof(cast('a' AS CHAR(2) COLLATE UTF8_LCASE)); +SELECT typeof(coalesce( + cast('a' AS CHAR(2) COLLATE UTF8_LCASE), cast('bb' AS CHAR(2) COLLATE UTF8_LCASE))); + +-- UNION LCT +SELECT typeof(c) FROM ( + SELECT cast('a' AS VARCHAR(3)) AS c + UNION ALL + SELECT cast('abcd' AS VARCHAR(8)) AS c +) t LIMIT 1; + +-- Nested types keep CHAR/VARCHAR +SELECT typeof(array(cast('a' AS CHAR(2)), cast('bb' AS CHAR(3)))); +SELECT typeof(struct(cast('a' AS CHAR(2)) AS f)); +SELECT typeof(map('k', cast('a' AS VARCHAR(2)))); + +-- R3: bare column references keep the declared type; write and read pad CHAR +CREATE TABLE char_varchar_std (c CHAR(5), v VARCHAR(5)) USING parquet; +INSERT INTO char_varchar_std VALUES ('ab', 'ab'); +SELECT typeof(c), typeof(v) FROM char_varchar_std; +SELECT concat('[', c, ']'), concat('[', v, ']') FROM char_varchar_std; +SELECT length(c), length(v) FROM char_varchar_std; +DROP TABLE char_varchar_std; diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar-standard-semantics.sql.out b/sql/core/src/test/resources/sql-tests/results/charvarchar-standard-semantics.sql.out new file mode 100644 index 0000000000000..1830a80cac51a --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/charvarchar-standard-semantics.sql.out @@ -0,0 +1,459 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT typeof(CAST('ab' AS CHAR(5))) +-- !query schema +struct +-- !query output +char(5) + + +-- !query +SELECT typeof(CAST('hello' AS VARCHAR(5))) +-- !query schema +struct +-- !query output +varchar(5) + + +-- !query +SELECT 'X' || CAST('5' AS CHAR(5)) || 'X' +-- !query schema +struct +-- !query output +X5 X + + +-- !query +SELECT CAST('ab ' AS CHAR(2)) +-- !query schema +struct +-- !query output +ab + + +-- !query +SELECT CAST('abcdef' AS CHAR(2)) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "EXCEED_LIMIT_LENGTH", + "sqlState" : "54006", + "messageParameters" : { + "limit" : "2" + } +} + + +-- !query +SELECT CAST('abcdef' AS VARCHAR(2)) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "EXCEED_LIMIT_LENGTH", + "sqlState" : "54006", + "messageParameters" : { + "limit" : "2" + } +} + + +-- !query +SELECT try_cast('abcdef' AS CHAR(2)) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT try_cast('abcdef' AS VARCHAR(2)) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world' AS VARCHAR(10)))) +-- !query schema +struct +-- !query output +varchar(10) + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), cast('world!' AS CHAR(6)))) +-- !query schema +struct +-- !query output +varchar(6) + + +-- !query +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), cast('world!' AS CHAR(6)))) +-- !query schema +struct +-- !query output +char(6) + + +-- !query +SELECT typeof(coalesce(cast('hello' AS VARCHAR(5)), 'world')) +-- !query schema +struct +-- !query output +string collate UTF8_BINARY + + +-- !query +SELECT typeof(coalesce(cast('hello' AS CHAR(5)), NULL)) +-- !query schema +struct +-- !query output +char(5) + + +-- !query +SELECT typeof( + CASE WHEN true THEN cast('a' AS CHAR(2)) ELSE cast('bb' AS CHAR(4)) END) +-- !query schema +struct +-- !query output +char(4) + + +-- !query +SELECT cast('a' AS CHAR(2)) IN (cast('a ' AS CHAR(2)), cast('bbb' AS VARCHAR(3))) +-- !query schema +struct<(CAST(a AS CHAR(2)) IN (CAST(a AS CHAR(2)), CAST(bbb AS VARCHAR(3)))):boolean> +-- !query output +true + + +-- !query +SELECT typeof(c) FROM (SELECT cast('a' AS CHAR(2)) AS c) t WHERE c IN ('a ', 'b') +-- !query schema +struct +-- !query output +char(2) + + +-- !query +SELECT typeof(upper(cast('ab' AS CHAR(2)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(lower(cast('AB' AS VARCHAR(2)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(cast('a' AS CHAR(1)) || cast('b' AS VARCHAR(1))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(substr(cast('hello' AS VARCHAR(5)), 1, 2)) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(upper(coalesce(cast('a' AS CHAR(2)), cast('b' AS CHAR(4))))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(concat(cast('a' AS CHAR(2)), cast('b' AS CHAR(3)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(trim(cast('ab ' AS CHAR(4)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(lpad(cast('ab' AS CHAR(2)), 5, 'x')) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(regexp_replace(cast('ab' AS CHAR(2)), 'a', 'x')) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(regexp_extract(cast('ab' AS VARCHAR(2)), '(a)', 1)) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(regexp_extract_all(cast('aab' AS VARCHAR(3)), '(a)', 1)) +-- !query schema +struct +-- !query output +array + + +-- !query +SELECT typeof(split(cast('a,b' AS CHAR(3)), ',')) +-- !query schema +struct +-- !query output +array + + +-- !query +SELECT typeof(mask(cast('ab' AS CHAR(2)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1)) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT concat('<', overlay(cast('ab' AS CHAR(5)) PLACING 'x' FROM 1), '>') +-- !query schema +struct):string> +-- !query output + + + +-- !query +SELECT typeof(elt(1, cast('ab' AS CHAR(5)), 'x')) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(right(cast('ab' AS CHAR(5)), 2)) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT concat('<', right(cast('ab' AS CHAR(5)), 2), '>') +-- !query schema +struct):string> +-- !query output +< > + + +-- !query +SELECT typeof(left(cast('ab' AS CHAR(5)), 2)) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(reverse(cast('ab' AS CHAR(5)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(hex(cast('ab' AS CHAR(5)))) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT hex(cast('ab' AS CHAR(5))) +-- !query schema +struct +-- !query output +6162202020 + + +-- !query +SELECT typeof(array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-')) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT concat('<', array_join(array(cast('ab' AS CHAR(5)), cast('cd' AS CHAR(5))), '-'), '>') +-- !query schema +struct):string> +-- !query output + + + +-- !query +SELECT typeof(reverse(array(1, 2))) +-- !query schema +struct +-- !query output +array + + +-- !query +SELECT typeof(str_to_map(cast('a:1,b:2' AS CHAR(7)))) +-- !query schema +struct +-- !query output +map + + +-- !query +SELECT typeof(c0) FROM (SELECT json_tuple(cast('{"a":"1"}' AS CHAR(9)), 'a') AS c0) +-- !query schema +struct +-- !query output +string + + +-- !query +SELECT typeof(cast('a' AS CHAR(2) COLLATE UTF8_LCASE)) +-- !query schema +struct +-- !query output +char(2) collate UTF8_LCASE + + +-- !query +SELECT typeof(coalesce( + cast('a' AS CHAR(2) COLLATE UTF8_LCASE), cast('bb' AS CHAR(2) COLLATE UTF8_LCASE))) +-- !query schema +struct +-- !query output +char(2) collate UTF8_LCASE + + +-- !query +SELECT typeof(c) FROM ( + SELECT cast('a' AS VARCHAR(3)) AS c + UNION ALL + SELECT cast('abcd' AS VARCHAR(8)) AS c +) t LIMIT 1 +-- !query schema +struct +-- !query output +varchar(8) + + +-- !query +SELECT typeof(array(cast('a' AS CHAR(2)), cast('bb' AS CHAR(3)))) +-- !query schema +struct +-- !query output +array + + +-- !query +SELECT typeof(struct(cast('a' AS CHAR(2)) AS f)) +-- !query schema +struct +-- !query output +struct + + +-- !query +SELECT typeof(map('k', cast('a' AS VARCHAR(2)))) +-- !query schema +struct +-- !query output +map + + +-- !query +CREATE TABLE char_varchar_std (c CHAR(5), v VARCHAR(5)) USING parquet +-- !query schema +struct<> +-- !query output + + + +-- !query +INSERT INTO char_varchar_std VALUES ('ab', 'ab') +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT typeof(c), typeof(v) FROM char_varchar_std +-- !query schema +struct +-- !query output +char(5) varchar(5) + + +-- !query +SELECT concat('[', c, ']'), concat('[', v, ']') FROM char_varchar_std +-- !query schema +struct +-- !query output +[ab ] [ab] + + +-- !query +SELECT length(c), length(v) FROM char_varchar_std +-- !query schema +struct +-- !query output +5 2 + + +-- !query +DROP TABLE char_varchar_std +-- !query schema +struct<> +-- !query output + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala index 8d22575f5d09b..34e268d3ba797 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala @@ -17,7 +17,10 @@ package org.apache.spark.sql -import org.apache.spark.{SparkConf, SparkRuntimeException} +import scala.util.Try + +import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException} +import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.catalyst.expressions.{Attribute, EqualTo, GreaterThan, ScalarSubquery, StringRPad} import org.apache.spark.sql.catalyst.expressions.Cast.toSQLId import org.apache.spark.sql.catalyst.parser.CatalystSqlParser @@ -27,6 +30,7 @@ import org.apache.spark.sql.connector.SchemaRequiredDataSource import org.apache.spark.sql.connector.catalog.{CatalogV2Util, InMemoryPartitionTableCatalog} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.functions import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.SimpleInsertSource import org.apache.spark.sql.test.SharedSparkSession @@ -854,6 +858,453 @@ class BasicCharVarcharTestSuite extends SharedSparkSession { } } + test("SPARK-58797: CAST to CHAR/VARCHAR with standardSemantics") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val charDf = sql("SELECT CAST('ab' AS CHAR(5)) AS c") + assert(charDf.schema.head.dataType === CharType(5)) + checkAnswer(charDf, Row("ab ")) + + val varcharDf = sql("SELECT CAST('hello' AS VARCHAR(5)) AS v") + assert(varcharDf.schema.head.dataType === VarcharType(5)) + checkAnswer(varcharDf, Row("hello")) + + checkError( + exception = intercept[SparkRuntimeException] { + sql("SELECT CAST('hello!' AS VARCHAR(5))").collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "5") + ) + + // Multi-byte characters: length is in characters, not octets. + // scalastyle:off nonascii + checkAnswer(sql("SELECT CAST('你好' AS VARCHAR(2)) AS v"), Row("你好")) + checkError( + exception = intercept[SparkRuntimeException] { + sql("SELECT CAST('你好啊' AS VARCHAR(2))").collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "2") + ) + // scalastyle:on nonascii + } + } + + test("SPARK-58798: least common type for COALESCE/CASE with CHAR/VARCHAR") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + assert(sql( + "SELECT coalesce(cast('hello' AS VARCHAR(5)), cast('world' AS VARCHAR(10))) AS c") + .schema.head.dataType === VarcharType(10)) + assert(sql( + "SELECT coalesce(cast('hello' AS VARCHAR(5)), cast('world!' AS CHAR(6))) AS c") + .schema.head.dataType === VarcharType(6)) + assert(sql( + "SELECT coalesce(cast('hello' AS CHAR(5)), cast('world!' AS CHAR(6))) AS c") + .schema.head.dataType === CharType(6)) + assert(sql( + "SELECT coalesce(cast('hello' AS VARCHAR(5)), 'world') AS c") + .schema.head.dataType === StringType) + assert(sql( + """SELECT CASE WHEN true THEN cast('a' AS CHAR(2)) + |ELSE cast('bb' AS CHAR(4)) END AS c""".stripMargin) + .schema.head.dataType === CharType(4)) + // LCT(NULL, T) = T + assert(sql("SELECT coalesce(null, cast('a' AS CHAR(5))) AS c") + .schema.head.dataType === CharType(5)) + } + } + + test("SPARK-58799: transforming string functions return STRING") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + assert(sql("SELECT upper(cast('ab' AS CHAR(2))) AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT lower(cast('AB' AS VARCHAR(2))) AS c") + .schema.head.dataType === StringType) + assert(sql( + "SELECT cast('a' AS CHAR(1)) || cast('b' AS VARCHAR(1)) AS c") + .schema.head.dataType === StringType) + // Pads from CHAR participate in the concatenated value. + checkAnswer( + sql("SELECT cast('he' AS CHAR(4)) || cast('llo' AS CHAR(3)) AS c"), + Row("he llo")) + assert(sql("SELECT substr(cast('hello' AS VARCHAR(5)), 1, 2) AS c") + .schema.head.dataType === StringType) + assert(sql( + "SELECT upper(coalesce(cast('a' AS CHAR(2)), cast('b' AS CHAR(4)))) AS c") + .schema.head.dataType === StringType) + } + } + + test("SPARK-58798: LCT preserves collation on CHAR/VARCHAR") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val c1 = CharType(2, "UTF8_LCASE") + val c2 = CharType(4, "UTF8_LCASE") + assert(StringHelper.tightestCommonString(c1, c2).contains(CharType(4, "UTF8_LCASE"))) + val v1 = VarcharType(3, "UTF8_LCASE") + val v2 = VarcharType(5, "UTF8_LCASE") + assert(StringHelper.tightestCommonString(v1, v2).contains(VarcharType(5, "UTF8_LCASE"))) + assert(StringHelper.tightestCommonString(c1, v2).contains(VarcharType(5, "UTF8_LCASE"))) + } + } + + test("SPARK-58799: regexp/mask/split return STRING under standardSemantics") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + assert(sql("SELECT regexp_replace(cast('ab' AS CHAR(2)), 'a', 'x') AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT regexp_extract(cast('ab' AS VARCHAR(2)), '(a)', 1) AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT split(cast('a,b' AS CHAR(3)), ',') AS c") + .schema.head.dataType === ArrayType(StringType, containsNull = false)) + assert(sql("SELECT mask(cast('ab' AS CHAR(2))) AS c") + .schema.head.dataType === StringType) + } + } + + test("SPARK-58796: preserve vs standardSemantics R1 matrix") { + // preserve-only: transforming ops may keep Char/Varchar (leaky experimental path). + withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") { + assert(sql("SELECT upper(cast('ab' AS CHAR(2))) AS c") + .schema.head.dataType === CharType(2)) + } + // standardSemantics: R1 forces STRING even if preserve is also on. + withSQLConf( + SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true", + SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + assert(sql("SELECT upper(cast('ab' AS CHAR(2))) AS c") + .schema.head.dataType === StringType) + } + } + + test("SPARK-58797: standardSemantics wins over charVarcharAsString") { + withSQLConf( + SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key -> "true", + SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val df = sql("SELECT CAST('ab' AS CHAR(5)) AS c") + assert(df.schema.head.dataType === CharType(5)) + checkAnswer(df, Row("ab ")) + } + } + + test("SPARK-58796: createDataFrame allows CHAR/VARCHAR when standardSemantics") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val df = spark.range(1).map(_.toString).toDF() + val schema = new StructType().add("id", CharType(5)) + val created = spark.createDataFrame(df.collectAsList(), schema) + assert(created.schema.head.dataType === CharType(5)) + checkAnswer(created, Row("0 ")) + + // RowEncoder must retain a declared collation on the constrained type, not rebuild + // CharType(length) / VarcharType(length) with the default collation. + val collated = new StructType() + .add("c", CharType(5, "UTF8_LCASE")) + .add("v", VarcharType(5, "UTF8_LCASE")) + val collatedDf = spark.createDataFrame( + java.util.Arrays.asList(Row("ab", "cd")), collated) + assert(collatedDf.schema("c").dataType === CharType(5, "UTF8_LCASE")) + assert(collatedDf.schema("v").dataType === VarcharType(5, "UTF8_LCASE")) + checkAnswer(collatedDf, Row("ab ", "cd")) + } + } + + test("SPARK-58803: Dataset/encoder/UDF CHAR/VARCHAR under standardSemantics") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + // createDataFrame / RowEncoder write-side: pad CHAR, reject oversize. + val charSchema = new StructType().add("c", CharType(3)) + checkAnswer( + spark.createDataFrame(java.util.Arrays.asList(Row("ab")), charSchema), + Row("ab ")) + checkError( + exception = intercept[SparkRuntimeException] { + spark.createDataFrame(java.util.Arrays.asList(Row("abcd")), charSchema).collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "3")) + val varcharSchema = new StructType().add("v", VarcharType(3)) + checkAnswer( + spark.createDataFrame(java.util.Arrays.asList(Row("ab")), varcharSchema), + Row("ab")) + // Oversize by trailing blanks only: trim just enough to fit the limit. + checkAnswer( + spark.createDataFrame(java.util.Arrays.asList(Row("abc ")), varcharSchema), + Row("abc")) + checkError( + exception = intercept[SparkRuntimeException] { + spark.createDataFrame(java.util.Arrays.asList(Row("abcd")), varcharSchema).collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "3")) + + // Explicit Encoders.CHAR / VARCHAR: typed Dataset write-side checks. + val charDs = spark.createDataset(Seq("ab"))(Encoders.CHAR(4)) + assert(charDs.schema.head.dataType === CharType(4)) + checkAnswer(charDs.toDF(), Row("ab ")) + checkError( + exception = intercept[SparkRuntimeException] { + spark.createDataset(Seq("abcde"))(Encoders.VARCHAR(3)).collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "3")) + + // UDF register: return type stays CHAR/VARCHAR; write-side pad / length apply. + spark.udf.register("std_char_udf", () => "B", CharType(3)) + spark.udf.register("std_varchar_udf", (x: String) => x, VarcharType(3)) + val charUdf = sql("SELECT std_char_udf() AS c") + assert(charUdf.schema.head.dataType === CharType(3)) + checkAnswer(charUdf, Row("B ")) + val varcharUdf = sql("SELECT std_varchar_udf('ab') AS v") + assert(varcharUdf.schema.head.dataType === VarcharType(3)) + checkAnswer(varcharUdf, Row("ab")) + checkError( + exception = intercept[SparkException] { + sql("SELECT std_varchar_udf('abcd')").collect() + }.getCause.asInstanceOf[SparkRuntimeException], + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "3")) + + // Java udf(..., returnType) path and Dataset.encoder from CHAR result schema. + val javaUdf = functions.udf( + new org.apache.spark.sql.api.java.UDF0[String] { + override def call(): String = "a" + }, + CharType(5)) + val javaUdfDf = spark.range(1).select(javaUdf().as("c")) + assert(javaUdfDf.schema.head.dataType === CharType(5)) + checkAnswer(javaUdfDf, Row("a ")) + assert(javaUdfDf.encoder.schema.head.dataType === CharType(5)) + + // Dataset.to: CHAR/VARCHAR target schema allowed; Cast applies store assignment. + withTable("std_cv_to") { + sql("CREATE TABLE std_cv_to (c CHAR(10), v VARCHAR(255)) USING parquet") + sql("INSERT INTO std_cv_to VALUES ('spark', 'awesome')") + val df = sql("SELECT * FROM std_cv_to") + assert(df.schema("c").dataType === CharType(10)) + assert(df.schema("v").dataType === VarcharType(255)) + val reordered = StructType.fromDDL("v VARCHAR(255), c CHAR(10)") + val toDf = df.to(reordered) + assert(toDf.schema.map(_.dataType) === Seq(VarcharType(255), CharType(10))) + checkAnswer(toDf, Row("awesome", "spark ")) + // Narrowing CHAR length is store assignment and must enforce length. + checkError( + exception = intercept[SparkRuntimeException] { + df.select($"c").to(new StructType().add("c", CharType(3))).collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "3")) + } + + // DataFrameReader / DataStreamReader user schemas keep CHAR/VARCHAR. + val readerSchema = new StructType().add("id", CharType(5)) + val csvInput = spark.range(1).map(_.toString) + val csvDf = spark.read.schema(readerSchema).csv(csvInput) + assert(csvDf.schema.head.dataType === CharType(5)) + checkAnswer(csvDf, Row("0 ")) + val csvDfDdl = spark.read.schema("id VARCHAR(5)").csv(csvInput) + assert(csvDfDdl.schema.head.dataType === VarcharType(5)) + withTempPath { dir => + spark.range(1).write.save(dir.toString) + val streamDf = spark.readStream.schema(readerSchema).load(dir.toString) + assert(streamDf.schema.head.dataType === CharType(5)) + val streamDdl = spark.readStream.schema("id VARCHAR(5)").load(dir.toString) + assert(streamDdl.schema.head.dataType === VarcharType(5)) + } + } + } + + test("SPARK-58794: R1 promotion unifies CHAR/VARCHAR with STRING at plain-string inputs") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + withTable("std_promote") { + sql("CREATE TABLE std_promote (c CHAR(5), v VARCHAR(5)) USING parquet") + sql("INSERT INTO std_promote VALUES ('ab', 'ab')") + + // Expressions requiring all their string inputs to share one type must accept a + // CHAR/VARCHAR argument alongside a STRING one by promoting it to STRING. + Seq( + "overlay(c PLACING 'x' FROM 1)" -> "xb ", + "overlay(v PLACING 'x' FROM 1)" -> "xb", + "string_agg(c, '-')" -> "ab ", + "listagg(c, '-')" -> "ab ", + "elt(1, c, 'x')" -> "ab ", + // right() is RuntimeReplaceable; its literal branches must agree with the substring + // branch, which R1 has already reduced to STRING. + "right(c, 2)" -> " ", + "left(c, 2)" -> "ab").foreach { case (expr, expected) => + val df = sql(s"SELECT $expr AS r FROM std_promote") + assert(df.schema.head.dataType === StringType, s"$expr should return STRING") + checkAnswer(df, Row(expected)) + } + + // Transforming expressions must not inherit the input's length constraint: each of these + // produces a value whose length differs from the CHAR(5) input. + Seq( + "reverse(c)" -> " ba", + "hex(c)" -> "6162202020", + "array_join(array(c, c), '-')" -> "ab -ab ").foreach { case (expr, expected) => + val df = sql(s"SELECT $expr AS r FROM std_promote") + assert(df.schema.head.dataType === StringType, s"$expr should return STRING") + checkAnswer(df, Row(expected)) + } + + // Promotion must not reach pass-through / LCT sites, which preserve CHAR/VARCHAR (R2/R3). + Seq( + "c", "coalesce(c, c)", "case when true then c else c end", "max(c)", + "element_at(array(c), 1)", "transform(array(c), x -> x)[0]", + "first_value(c) over (order by 1)").foreach { expr => + val df = sql(s"SELECT $expr AS r FROM std_promote") + assert(df.schema.head.dataType === CharType(5), s"$expr should stay CHAR(5)") + } + + // reverse() on non-string inputs is unaffected by the R1 change. + assert(sql("SELECT reverse(array(1, 2)) AS r").schema.head.dataType === + ArrayType(IntegerType, containsNull = false)) + } + } + } + + // Allowlist for the inventory below: R2/R3 pass-through and container cases that may keep + // CHAR(n)/VARCHAR(n): aggregates/ordering that return an input unchanged, null-handling, + // element access, array/map/struct constructors, and collection rearrangements that keep + // element types. Coverage is limited to the seven fixed argumentShapes templates in the test; + // a leak only at another arity or nested shape would not fail here. For those shapes, + // anything not listed must reduce to plain STRING (R1). + private val charVarcharPassThroughFunctions = Set( + "any_value", "approx_top_k", "approx_top_k_accumulate", "array", "array_agg", "array_compact", + "array_distinct", "array_max", "array_min", "array_repeat", "array_sort", "arrays_zip", + "coalesce", "collect_list", "collect_set", "collect_union", "concat", "explode", + "explode_outer", "first", "first_value", "get", "greatest", "ifnull", "last", "last_value", + "least", "map", "max", "max_by", "measure", "min", "min_by", "mode", "named_struct", "nullif", + "nullifzero", "nvl", "reverse", "shuffle", "sort_array", "struct", "when") + + test("SPARK-58794: inventoried shapes do not leak CHAR/VARCHAR under standardSemantics") { + val argumentShapes = Seq( + "%s(c)", "%s(c, c)", "%s(c, 'x')", "%s('x', c)", "%s(c, 1)", "%s(array(c))", + "%s(array(c), '-')") + + withTable("std_inventory") { + sql("CREATE TABLE std_inventory (c CHAR(5)) USING parquet") + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + val leaks = FunctionRegistry.functionSet.map(_.funcName).toSeq.sorted + .filterNot(charVarcharPassThroughFunctions.contains) + .flatMap { name => + argumentShapes.map(_.format(name)).filter { call => + // Most shapes do not typecheck for a given function; those are simply not evidence. + Try(sql(s"SELECT $call AS r FROM std_inventory").schema.head.dataType) + .toOption + .exists(CharVarcharUtils.hasCharVarchar) + } + } + + assert(leaks.isEmpty, + "these inventoried calls returned a CHAR/VARCHAR type; either fix the expression to " + + "return plain STRING or add it to charVarcharPassThroughFunctions: " + + leaks.mkString(", ")) + } + } + } + + test("SPARK-58802: single-pass resolver agrees with fixed-point under standardSemantics") { + // Dual run defaults to on under tests, but pin it explicitly so this coverage cannot be + // silently lost: the HybridAnalyzer compares output schema and normalized plan across the + // two analyzers and fails with HYBRID_ANALYZER_EXCEPTION on any divergence. Resolver has no + // Char/Varchar-specific logic; it inherits Expression.dataType and shared TypeCoercion, so + // this matrix is the proof that LCT/CAST/R1 stay aligned (D19). + withSQLConf( + SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true", + SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key -> "true", + SQLConf.ANALYZER_DUAL_RUN_SAMPLE_RATE.key -> "1.0", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY.key -> "false", + SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_EXPOSE_RESOLVER_GUARD_FAILURE.key -> "true") { + // CAST / try_cast introduce the type (R3). + assert(sql("SELECT CAST('ab' AS CHAR(5)) AS c").schema.head.dataType === CharType(5)) + assert(sql("SELECT CAST('hello' AS VARCHAR(5)) AS c").schema.head.dataType === + VarcharType(5)) + assert(sql("SELECT try_cast('abcdef' AS CHAR(2)) AS c").schema.head.dataType === + CharType(2)) + checkAnswer(sql("SELECT try_cast('abcdef' AS VARCHAR(2)) AS c"), Row(null)) + + // Least common type (R2): COALESCE / CASE / NULL / CHAR+VARCHAR / CHAR+STRING. + assert(sql( + "SELECT coalesce(CAST('a' AS VARCHAR(3)), CAST('bb' AS VARCHAR(7))) AS c") + .schema.head.dataType === VarcharType(7)) + assert(sql( + "SELECT coalesce(CAST('a' AS CHAR(2)), CAST('bb' AS VARCHAR(4))) AS c") + .schema.head.dataType === VarcharType(4)) + assert(sql( + "SELECT coalesce(CAST('a' AS CHAR(2)), 'bb') AS c") + .schema.head.dataType === StringType) + assert(sql( + "SELECT coalesce(CAST('a' AS CHAR(5)), CAST(NULL AS CHAR(5))) AS c") + .schema.head.dataType === CharType(5)) + assert(sql( + "SELECT CASE WHEN true THEN CAST('a' AS CHAR(2)) ELSE CAST('bb' AS CHAR(4)) END AS c") + .schema.head.dataType === CharType(4)) + assert(sql( + "SELECT CASE WHEN false THEN CAST('a' AS VARCHAR(2)) ELSE CAST('bb' AS CHAR(4)) END AS c") + .schema.head.dataType === VarcharType(4)) + + // IN-list common type (side condition uses LCT; result is boolean). + checkAnswer( + sql("SELECT CAST('a' AS CHAR(2)) IN (CAST('a ' AS CHAR(2)), CAST('bbb' AS VARCHAR(3)))"), + Row(true)) + + // Transforming operators return STRING (R1). + assert(sql("SELECT upper(CAST('ab' AS CHAR(2))) AS c").schema.head.dataType === StringType) + assert(sql("SELECT lower(CAST('AB' AS VARCHAR(2))) AS c").schema.head.dataType === + StringType) + assert(sql("SELECT CAST('a' AS CHAR(1)) || CAST('b' AS VARCHAR(1)) AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT concat(CAST('a' AS CHAR(2)), CAST('b' AS CHAR(3))) AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT substr(CAST('hello' AS VARCHAR(5)), 1, 2) AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT trim(CAST('ab ' AS CHAR(4))) AS c").schema.head.dataType === StringType) + assert(sql("SELECT regexp_replace(CAST('ab' AS CHAR(2)), 'a', 'x') AS c") + .schema.head.dataType === StringType) + assert(sql("SELECT mask(CAST('ab' AS CHAR(2))) AS c").schema.head.dataType === StringType) + assert(sql("SELECT split(CAST('a,b' AS CHAR(3)), ',') AS c").schema.head.dataType === + ArrayType(StringType, containsNull = false)) + // R1 after LCT: coalesce stays CHAR, upper widens to STRING. + assert(sql( + "SELECT upper(coalesce(CAST('a' AS CHAR(2)), CAST('b' AS CHAR(4)))) AS c") + .schema.head.dataType === StringType) + + // Set-operation LCT. + val union = sql( + """SELECT CAST('a' AS VARCHAR(3)) AS c + |UNION ALL + |SELECT CAST('abcd' AS VARCHAR(8)) AS c""".stripMargin) + assert(union.schema.head.dataType === VarcharType(8)) + checkAnswer(union, Seq(Row("a"), Row("abcd"))) + + val intersect = sql( + """SELECT CAST('ab' AS CHAR(2)) AS c + |INTERSECT + |SELECT CAST('ab' AS CHAR(4)) AS c""".stripMargin) + assert(intersect.schema.head.dataType === CharType(4)) + checkAnswer(intersect, Seq(Row("ab "))) + + // Nested types keep CHAR/VARCHAR through analysis. + assert(sql("SELECT array(CAST('a' AS CHAR(2)), CAST('bb' AS CHAR(3))) AS c") + .schema.head.dataType === ArrayType(CharType(3), containsNull = false)) + assert(sql("SELECT struct(CAST('a' AS CHAR(2)) AS f) AS c") + .schema.head.dataType === + StructType(Seq(StructField("f", CharType(2), nullable = false)))) + + // Collated CAST keeps the declared collation under both analyzers. + assert(sql("SELECT CAST('ab' AS CHAR(2) COLLATE UTF8_LCASE) AS c") + .schema.head.dataType === CharType(2, "UTF8_LCASE")) + + // Bare column references keep the declared type (R3) through dual-run analysis. + withTable("char_varchar_dual_run") { + sql("CREATE TABLE char_varchar_dual_run (c CHAR(5), v VARCHAR(5)) USING parquet") + sql("INSERT INTO char_varchar_dual_run VALUES ('ab', 'ab')") + val df = sql("SELECT c, v FROM char_varchar_dual_run") + assert(df.schema("c").dataType === CharType(5)) + assert(df.schema("v").dataType === VarcharType(5)) + checkAnswer(df, Row("ab ", "ab")) + } + } + } + test("invalidate char/varchar in functions") { checkError( exception = intercept[AnalysisException] { @@ -1040,6 +1491,41 @@ class FileSourceCharVarcharTestSuite extends CharVarcharTestSuite with SharedSpa } } + test("SPARK-58801: standardSemantics scan pads CHAR and errors on oversize") { + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + withTempPath { dir => + withTable("t") { + sql("SELECT '12' as col").write.format(format).save(dir.toString) + sql(s"CREATE TABLE t (col CHAR(3)) using $format LOCATION '$dir'") + checkAnswer(sql("SELECT * FROM t"), Row("12 ")) + } + } + Seq("CHAR", "VARCHAR").foreach { typ => + withTempPath { dir => + withTable("t") { + sql("SELECT '123456' as col").write.format(format).save(dir.toString) + sql(s"CREATE TABLE t (col $typ(2)) using $format LOCATION '$dir'") + checkError( + exception = intercept[SparkRuntimeException] { + sql("SELECT * FROM t").collect() + }, + condition = "EXCEED_LIMIT_LENGTH", + parameters = Map("limit" -> "2") + ) + } + } + } + // An oversized value consisting only of trailing blanks is trimmed successfully. + withTempPath { dir => + withTable("t") { + sql("SELECT '12 ' as col").write.format(format).save(dir.toString) + sql(s"CREATE TABLE t (col VARCHAR(2)) using $format LOCATION '$dir'") + checkAnswer(sql("SELECT * FROM t"), Row("12")) + } + } + } + } + test("alter table set location w/ fit length values") { withTempPath { dir => withTable("t") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala index 4e39f01bf5374..21384d7c4b917 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala @@ -1252,6 +1252,21 @@ class UDFSuite extends SharedSparkSession { parameters = Map("dataType" -> s"\"${dt.sql}\"") ) } + withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") { + Seq(CharType(5), VarcharType(5)).foreach { dt => + val f = udf( + new UDF0[String] { + override def call(): String = "a" + }, + dt + ) + val df = spark.range(1).select(f().as("c")) + assert(df.schema.head.dataType === dt) + assert(df.encoder.schema.head.dataType === dt) + val expected = if (dt.isInstanceOf[CharType]) "a " else "a" + checkAnswer(df, Row(expected)) + } + } } test("SPARK-47927: ScalaUDF null handling") { diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtils.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtils.scala index df31ca311d46b..c4a7d1727bdcc 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtils.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtils.scala @@ -147,8 +147,12 @@ object RowSetUtils { // types that reach this branch do not use the `nested` flag in `toHiveString`. Now, // Geospatial types use it for wrapping EWKT in quotes when nested = true, so we need // to set `nested` here to false to avoid spurious quotes for standalone geo values. + // String types need the same treatment: the fast path above matches only the + // default-collation StringType singleton, so CHAR/VARCHAR and collated strings land + // here and would otherwise be rendered as "value" instead of value. val nested = typ match { case _: GeometryType | _: GeographyType => false + case _: StringType => false case _ => true } toHiveString((row.get(ordinal), typ), nested, timeFormatters, binaryFormatter) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtilsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtilsSuite.scala new file mode 100644 index 0000000000000..b2f0d462a3a7b --- /dev/null +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/RowSetUtilsSuite.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.thriftserver + +import scala.jdk.CollectionConverters._ + +import org.apache.hive.service.rpc.thrift.TProtocolVersion + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.Row +import org.apache.spark.sql.types.{CharType, DataType, StringType, VarcharType} + +class RowSetUtilsSuite extends SparkFunSuite { + + private def stringValues(value: String, dataType: DataType): Seq[String] = { + val rowSet = RowSetUtils.toTRowSet( + 0, + Seq(Row(value)), + Array(dataType), + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V8) + rowSet.getColumns.asScala.head.getStringVal.getValues.asScala.toSeq + } + + // Only the default-collation StringType singleton takes the fast path in toTColumn. CHAR, + // VARCHAR and collated strings fall through to the generic branch, which renders values with + // toHiveString and used to quote them there. + test("SPARK-58794: CHAR, VARCHAR and collated string values are not quoted") { + Seq( + CharType(4), + VarcharType(4), + StringType("UTF8_LCASE"), + StringType).foreach { dataType => + assert(stringValues("ab", dataType) === Seq("ab"), s"$dataType was rendered with quotes") + } + } +}